diff --git a/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py new file mode 100644 index 0000000000..13b002cda0 --- /dev/null +++ b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Routing/selection tests for the AITER ASM cprr DCP-verify route. + +Companion to the upstream segmented-verify tests. Everything here is pure CPU: +it pins the *decisions* (which route a KV group takes, which head count the +kernel runs at, which configurations are refused) rather than kernel numerics. +The numerics parity test against the segmented route needs a GPU and both +routes built; it is marked and skipped when unavailable. +""" + +from __future__ import annotations + +import pytest + +rocm_aiter_mla = pytest.importorskip( + "vllm.v1.attention.backends.mla.rocm_aiter_mla", + reason="ROCm AITER MLA backend not available", +) + +_heads = rocm_aiter_mla._asm_dcp_verify_heads +_configured = rocm_aiter_mla._asm_dcp_verify_configured +_select_route = rocm_aiter_mla._select_dcp_decode_route +Route = rocm_aiter_mla._DCPDecodeRoute +NATIVE = rocm_aiter_mla._NATIVE_CPRR_HEADS +MIN_QLEN = rocm_aiter_mla._MIN_CPRR_QLEN + + +def test_dcp_verify_env_defaults_to_segmented(monkeypatch): + monkeypatch.delenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", raising=False) + assert rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "segmented" + + +def test_dcp_verify_env_rejects_unknown_route(monkeypatch): + monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "unknown") + with pytest.raises(ValueError, match="Invalid value"): + _ = rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY + + +# -------------------------------------------------------------------------- +# _asm_dcp_verify_heads -- which head count the cprr kernel runs at +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n", NATIVE) +def test_native_head_counts_are_used_as_is(n): + assert _heads(n) == n + + +def test_non_native_head_count_pads_up_to_the_next_native(): + """K3 at TP8/DCP8 gathers 96 heads for the target; the kernel has no 96 + variant, so it must run at 128, not silently fall out of the asm path.""" + assert _heads(96) == 128 + assert _heads(1) == 16 + assert _heads(17) == 32 + assert _heads(65) == 128 + + +def test_head_count_above_the_largest_native_is_unservable(): + """0 means 'cannot serve'; anything else would route a shape the kernel + has no build for.""" + assert _heads(max(NATIVE) + 1) == 0 + assert _heads(256) == 0 + + +# -------------------------------------------------------------------------- +# _asm_dcp_verify_configured -- reachability +# -------------------------------------------------------------------------- + + +@pytest.fixture +def on_gfx950(monkeypatch): + """The cprr kernels are built only for gfx950, so the gate checks the arch. + Patched rather than detected, so both branches run on any runner.""" + import vllm.platforms.rocm as rocm + + monkeypatch.setattr(rocm, "on_gfx950", lambda: True) + + +def test_route_needs_dcp(on_gfx950): + assert _configured(dcp_world_size=1, cp_interleave=1, multi_token_decode=True) is ( + False + ) + assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is ( + True + ) + + +def test_round_robin_interleave_other_than_one_is_excluded(on_gfx950): + """The kernel's global-position causal window assumes interleave 1, the + same restriction the segmented gate carries.""" + assert _configured(dcp_world_size=8, cp_interleave=2, multi_token_decode=True) is ( + False + ) + assert _configured(dcp_world_size=8, cp_interleave=16, multi_token_decode=True) is ( + False + ) + + +def test_route_needs_multi_token_decode(on_gfx950): + """Single-token decode is already served. Enabling the route for it would + pad the head count on every step and buy nothing.""" + assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=False) is ( + False + ) + + +def test_route_needs_gfx950(monkeypatch): + """hsa/gfx942/mla and hsa/gfx1250/mla carry no cprr rows, so off gfx950 the + kernel lookup fails at the first verify step rather than falling back.""" + import vllm.platforms.rocm as rocm + + monkeypatch.setattr(rocm, "on_gfx950", lambda: False) + assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is ( + False + ) + + +# -------------------------------------------------------------------------- +# qlen floor +# -------------------------------------------------------------------------- + + +def test_min_cprr_qlen_is_above_two(): + """Qlen 2 is below CPRR's floor and must use segmented verification.""" + assert MIN_QLEN > 2 + + +@pytest.mark.parametrize( + "supports,causal,qlen,asm,expected", + [ + (True, True, 1, True, Route.PLAIN), + (True, True, 2, True, Route.SEGMENTED), + (True, True, 3, True, Route.CPRR), + (True, True, 4, True, Route.CPRR), + (True, True, 4, False, Route.SEGMENTED), + (True, False, 2, True, Route.PLAIN), + (True, False, 4, False, Route.PLAIN), + (False, True, 3, True, Route.CPRR), + ], +) +def test_dcp_decode_route(supports, causal, qlen, asm, expected): + assert _select_route(supports, causal, qlen, asm) is expected + + +def test_causal_multi_token_batch_without_a_valid_route_fails(): + with pytest.raises(RuntimeError, match="requires either segmented MLA"): + _select_route(False, True, 2, True) + + +# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs +# the real builder and the real asm kernel on a gfx950 GPU. diff --git a/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr_numerics.py b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr_numerics.py new file mode 100644 index 0000000000..44323dde41 --- /dev/null +++ b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr_numerics.py @@ -0,0 +1,540 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""GPU numerics for the AITER ASM cprr DCP-verify route. + +Runs the REAL metadata builder (``AiterMLAMetadataBuilder.build``, so +``get_mla_metadata_v1`` is exercised with the cprr kwargs and the persistent +schedule buffers are produced) and the REAL asm kernel, on ONE GPU with the DCP +ranks simulated one after another. + +That is legitimate because the aiter MLA backend contains no collective on this +path: the query all-gather happens in the *layer*, strictly before +``forward_mqa``, so the head fold, the round-robin kernel call and the un-fold +are pure per-rank local arithmetic in which ``cp_world_size``/``cp_rank`` are +just numbers feeding the causal-mask maths. Restricting a shard to a residue +class of global positions is exactly what DCP rank r sees. + +Three properties are checked: + * each shard matches an exact torch reference over the positions it holds, + with causality applied on the GLOBAL position (the thing cprr reconstructs + in-kernel and the segmented route instead carries in per-row lengths); + * the LSE merge of all shards reproduces full-context attention; + * a shard told the wrong ``cp_rank`` fails both -- without this positive + control a passing run proves nothing, since a kernel that ignored global + positions still produces plausible-looking output. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.platforms import current_platform + +if not current_platform.is_rocm(): + pytest.skip("ROCm AITER MLA tests", allow_module_level=True) + +from vllm._aiter_ops import is_aiter_found # noqa: E402 +from vllm.v1.attention.backends.mla import rocm_aiter_mla # noqa: E402 + +KV_LORA_RANK = 512 +QK_ROPE_HEAD_DIM = 64 +QK_NOPE_HEAD_DIM = 128 +HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM +SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5 + +DCP = 8 +NUM_REQS = 8 +CTX = 256 +QLEN = 5 # 1 + num_speculative_tokens; must clear _MIN_CPRR_QLEN + +# fp8-e4m3 keeps 3 mantissa bits, so the relative error floor is ~6.2e-2. Gates +# are relative for the same reason: an absolute bound would just be a bound on +# the output magnitude. +SHARD_RTOL = 1.0e-1 +MERGE_RTOL = 5.0e-2 + +MODEL = "deepseek-ai/DeepSeek-R1" + + +def _gpu_available() -> bool: + return is_aiter_found() and torch.accelerator.is_available() + + +def _on_gfx950() -> bool: + if not is_aiter_found(): + return False + from vllm.platforms.rocm import on_gfx950 + + return on_gfx950() + + +requires_cprr = pytest.mark.skipif( + not (_gpu_available() and _on_gfx950()), + reason="cprr DCP verify needs ROCm + AITER on gfx950", +) + + +@pytest.fixture(autouse=True) +def select_asm_route(monkeypatch): + monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "asm") + + +# -------------------------------------------------------------------------- +# reference maths +# -------------------------------------------------------------------------- + + +def _torch_reference(q_f32, kv_f32, ctx, qlen, scale, positions): + """Exact absorbed-MLA decode over an explicit set of GLOBAL KV positions. + + A shard sees only ``positions``, but causality still uses the global + position: query token i (global position ctx + i) may attend to any p in + positions with p <= ctx + i. Restricting ``positions`` to a round-robin + residue class is precisely DCP rank r's view. + """ + nr, _, heads, _ = q_f32.shape + dev = q_f32.device + o = torch.zeros(nr, qlen, heads, KV_LORA_RANK, dtype=torch.float32, device=dev) + lse = torch.full((nr, qlen, heads), float("-inf"), dtype=torch.float32, device=dev) + for r in range(nr): + kv_sel = kv_f32[r].index_select(0, positions) + for i in range(qlen): + valid = positions <= (ctx + i) + if not bool(valid.any()): + continue + k = kv_sel[valid] + s = (q_f32[r, i] @ k.T) * scale + m = s.max(dim=-1, keepdim=True).values + p = torch.exp(s - m) + denom = p.sum(dim=-1, keepdim=True) + o[r, i] = (p / denom) @ k[:, :KV_LORA_RANK] + lse[r, i] = m.squeeze(-1) + torch.log(denom.squeeze(-1)) + return o, lse + + +def _merge_partials(os_, lses): + """Cross-shard softmax merge (natural-log LSE).""" + stack_lse = torch.stack(lses) + m = stack_lse.max(dim=0).values + w = torch.exp(stack_lse - m) + num = (w.unsqueeze(-1) * torch.stack(os_)).sum(dim=0) + den = w.sum(dim=0).unsqueeze(-1) + return num / den, m + torch.log(w.sum(dim=0)) + + +# -------------------------------------------------------------------------- +# single-process rank simulation +# -------------------------------------------------------------------------- + + +class _FakeGroup: + def __init__(self, world_size: int, rank: int): + self.world_size = world_size + self.rank_in_group = rank + self.rank = rank + self.local_rank = rank + + def all_gather(self, tensor, dim=0): + raise RuntimeError( + "collective called in a single-process test -- the path under test " + "is supposed to be collective-free; this indicates a real change" + ) + + +def _install_fake_groups(dcp_size: int, cp_rank: int, tp_size: int = 1) -> None: + """Point vLLM's parallel_state at rank-simulating stubs (no NCCL).""" + import vllm.distributed.parallel_state as ps + import vllm.model_executor.layers.attention.mla_attention as mla_attention + import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend + + dcp = _FakeGroup(dcp_size, cp_rank) + tp = _FakeGroup(tp_size, cp_rank) + ps.get_dcp_group = lambda: dcp + ps.get_tp_group = lambda: tp + # Both of these imported the symbol directly at module import time, so + # patching parallel_state alone leaves them resolving the real group (and + # the builder then reports dcp_world_size 1, silently skipping the route). + backend.get_dcp_group = lambda: dcp + mla_attention.get_dcp_group = lambda: dcp + + +@pytest.fixture(autouse=True) +def _restore_parallel_groups(): + """Undo _install_fake_groups after every test. + + It patches module globals in three modules, so without this the fake DCP + group leaks into the rest of the pytest session and silently breaks + unrelated tests that expect a real (or absent) group. + """ + import vllm.distributed.parallel_state as ps + import vllm.model_executor.layers.attention.mla_attention as mla_attention + import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend + + saved = [ + (mod, name, getattr(mod, name)) + for mod, name in ( + (ps, "get_dcp_group"), + (ps, "get_tp_group"), + (backend, "get_dcp_group"), + (mla_attention, "get_dcp_group"), + ) + ] + yield + for mod, name, value in saved: + setattr(mod, name, value) + + +class _PrefillBackendStub: + """The base builder does ``attention_layer.prefill_backend.clone()``. + + prefill_backend is consulted only from the chunked-context prefill path, + which a decode-only test never enters. + """ + + def clone(self): + return self + + +def _make_ctx_layer(vllm_config): + """Stand-in for the registered attention layer in static_forward_context. + + ``MLACommonMetadataBuilder.__init__`` reads the MLA dims, ``dcp_manager`` + and ``prefill_backend`` off this object. ``dcp_manager`` is the Direct-DCP + symmetric-memory workspace, used ONLY by the chunked-prefill KV all-gather; + it is constructed without running __init__ so no peer pointers are exchanged + and no collective is issued. That is what keeps this test single-process. + """ + from vllm.v1.attention.ops.dcp import MLADCPManager + + mgr = object.__new__(MLADCPManager) + mgr.init_kv_gather = lambda *a, **k: None + + return SimpleNamespace( + non_causal_multi_token_decode=False, + q_lora_rank=None, + kv_lora_rank=KV_LORA_RANK, + qk_nope_head_dim=QK_NOPE_HEAD_DIM, + qk_rope_head_dim=QK_ROPE_HEAD_DIM, + v_head_dim=QK_NOPE_HEAD_DIM, + dcp_manager=mgr, + prefill_backend=_PrefillBackendStub(), + ) + + +def _ensure_workspace_manager(device: torch.device) -> None: + """Init the global workspace manager, which GPUModelRunner does in production. + + ``AiterMLAMetadataBuilder.__init__`` reserves the fp8 prefill + persistent-scheduling scratch through ``current_workspace_manager()``, so + merely constructing a builder outside a serve trips its assert. Idempotent: + one builder is constructed per DCP shard. Deliberately does NOT lock the + workspace -- production locks only after warmup, and this keeps allocating + across shards. + """ + from vllm.v1.worker.workspace import ( + init_workspace_manager, + is_workspace_manager_initialized, + ) + + if not is_workspace_manager_initialized(): + init_workspace_manager(device) + + +class _Layer: + def __init__(self, device): + self._q_scale = torch.tensor(1.0, device=device) + self._k_scale = torch.tensor(1.0, device=device) + self._v_scale = torch.tensor(1.0, device=device) + + +def _build_impl(num_heads: int, dcp: int, cp_rank: int): + """Real AiterMLAImpl.forward_mqa bound to a minimally-populated instance. + + Constructing the full impl would drag in kv_b_proj and the rest of the + weight-dependent MLA setup -- i.e. model loading, which this test exists to + avoid. forward_mqa's fold path reads only the attributes set below. + """ + from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl + + impl = object.__new__(AiterMLAImpl) + impl.kv_lora_rank = KV_LORA_RANK + impl.qk_rope_head_dim = QK_ROPE_HEAD_DIM + impl.num_heads = num_heads + impl.dcp_world_size = dcp + impl.dcp_rank = cp_rank + impl.pcp_world_size = 1 + impl.kv_cache_dtype = "fp8" + impl.scale = SCALE + # `_decode_num_heads` is a read-only property deriving num_heads * + # dcp_world_size, both set above; assert rather than assign. + assert impl._decode_num_heads == num_heads * dcp + return impl + + +def _make_vllm_config(heads_per_rank: int): + """A DCP8 MLA config with no weights and no distributed init. + + DeepSeek-R1's config supplies the MLA dims (kv_lora_rank 512, rope 64, + nope 128) that the aiter asm kernel is built for; only the head count is + overridden, to place the GATHERED count (heads_per_rank * DCP) on the + kernel variant under test. ``get_num_attention_heads`` reads + model_arch_config, a snapshot taken at config-creation time, so patching + hf_config alone would be inert. + """ + from tests.v1.attention.utils import create_vllm_config + + vllm_config = create_vllm_config( + model_name=MODEL, + tensor_parallel_size=1, + max_model_len=2048, + max_num_seqs=NUM_REQS * 2, + max_num_batched_tokens=4096, + ) + vllm_config.model_config.model_arch_config.total_num_attention_heads = ( + heads_per_rank + ) + # Without a speculative config the batch reorder threshold stays at 1 and a + # qlen-QLEN row is classified as a PREFILL, so the decode path under test is + # never reached. _init_reorder_batch_threshold reads exactly these two + # fields, so a stand-in is enough and avoids pulling in a draft model. + vllm_config.speculative_config = SimpleNamespace( + num_speculative_tokens=QLEN - 1, parallel_drafting=False + ) + vllm_config.cache_config.cache_dtype = "fp8" + vllm_config.parallel_config.decode_context_parallel_size = DCP + vllm_config.parallel_config.cp_kv_cache_interleave_size = 1 + return vllm_config + + +def _run_shard( + vllm_config, + spec, + layer_name, + device, + cp_rank, + kern_rank, + q_flat, + kv_flat, + nr, + qlen, + ctx, + dcp, + pos, +): + """Build real metadata for one DCP rank and run the real asm kernel. + + ``kern_rank`` is separate from ``cp_rank`` so the positive control can hand + rank r's pages to the kernel while telling it that it is rank r+1. + """ + from vllm.v1.attention.backend import CommonAttentionMetadata + from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder + + L = ctx + qlen + local_len = int(pos.numel()) + + # One GLOBAL kv cache: page (req*L + p) holds request req position p. Shards + # select pages by residue class, so every shard reads the same underlying + # bytes and no re-quantisation noise is introduced between arms. + block_table = ( + torch.arange(nr, device=device).view(nr, 1) * L + pos.view(1, -1) + ).to(torch.int32) + + # seq_lens is the GLOBAL context; dcp_local_seq_lens is what this rank + # actually holds. Passing the global length as the local one walks the + # kernel off the end of the page table. + seq_lens_cpu = torch.full((nr,), L, dtype=torch.int32) + local_cpu = torch.full((nr,), local_len, dtype=torch.int32) + + common = CommonAttentionMetadata( + query_start_loc=torch.arange( + 0, nr * qlen + 1, qlen, dtype=torch.int32, device=device + ), + query_start_loc_cpu=torch.arange(0, nr * qlen + 1, qlen, dtype=torch.int32), + seq_lens=seq_lens_cpu.to(device), + num_reqs=nr, + num_actual_tokens=nr * qlen, + max_query_len=qlen, + max_seq_len=L, + block_table_tensor=block_table, + slot_mapping=torch.arange(nr * qlen, dtype=torch.int64, device=device), + causal=True, + dcp_local_seq_lens=local_cpu.to(device), + dcp_local_seq_lens_cpu_upper_bound=local_cpu, + ) + + _install_fake_groups(dcp, kern_rank) + builder = AiterMLAMetadataBuilder(spec, [layer_name], vllm_config, device) + md = builder.build(0, common) + + impl = _build_impl(builder.num_heads, dcp, kern_rank) + o, lse = impl.forward_mqa(q_flat, kv_flat, md, _Layer(device)) + torch.accelerator.synchronize() + return md, o, lse, local_len + + +def _numerics(heads_per_rank: int, sabotage: bool = False): + """Run every DCP shard and return (per-shard max rel err, merged rel err).""" + import aiter + + from vllm.config import set_current_vllm_config + + device = torch.device("cuda", 0) + torch.accelerator.set_device_index(device.index) + torch.manual_seed(0) + _ensure_workspace_manager(device) + + _install_fake_groups(DCP, 0) + vllm_config = _make_vllm_config(heads_per_rank) + + with set_current_vllm_config(vllm_config): + from vllm.v1.kv_cache_interface import MLAAttentionSpec + + layer_name = "model.layers.0.self_attn.attn" + vllm_config.compilation_config.static_forward_context[layer_name] = ( + _make_ctx_layer(vllm_config) + ) + # block_size 1: one page per token, so page ids are positions and the + # residue-class page table is expressible directly. + spec = MLAAttentionSpec( + block_size=1, + num_kv_heads=1, + head_size=HEAD_SIZE, + dtype=vllm_config.model_config.dtype, + non_causal_multi_token_decode=False, + ) + + nr, ctx, qlen = NUM_REQS, CTX, QLEN + L = ctx + qlen + gathered_heads = heads_per_rank * DCP + kv_dtype = aiter.dtypes.fp8 + + kv_flat = torch.randn( + nr * L, 1, HEAD_SIZE, dtype=torch.float32, device=device + ).to(kv_dtype) + q_flat = ( + torch.randn( + nr * qlen, + gathered_heads, + HEAD_SIZE, + dtype=torch.float32, + device=device, + ) + .mul_(0.5) + .to(kv_dtype) + ) + # Dequantise for the reference so the comparison measures the kernel, + # not the fp8 cast that both sides share. + kv_f32 = kv_flat.float().view(nr, L, HEAD_SIZE) + q_f32 = q_flat.float().view(nr, qlen, gathered_heads, HEAD_SIZE) + + ref_o, _ = _torch_reference( + q_f32, kv_f32, ctx, qlen, SCALE, torch.arange(L, device=device) + ) + + shard_o, shard_lse = [], [] + max_shard_rel = 0.0 + for r in range(DCP): + pos = torch.arange(r, L, DCP, device=device) # residue class r + kern_rank = (r + 1) % DCP if sabotage else r + md, o, lse, _ = _run_shard( + vllm_config, + spec, + layer_name, + device, + r, + kern_rank, + q_flat, + kv_flat, + nr, + qlen, + ctx, + DCP, + pos, + ) + # The route actually under test must have been taken; a silent + # fall-out to Triton/segmented would otherwise pass quietly. + assert md.decode.dcp_route is rocm_aiter_mla._DCPDecodeRoute.CPRR + assert md.decode.asm_decode_num_heads, "cprr asm route was not taken" + + o = o.float().view(nr, qlen, gathered_heads, KV_LORA_RANK) + shard_lse.append(lse.float().view(nr, qlen, gathered_heads)) + shard_o.append(o) + + po, _ = _torch_reference(q_f32, kv_f32, ctx, qlen, SCALE, pos) + rel = (o - po).abs().max().item() / max(po.abs().max().item(), 1e-6) + max_shard_rel = max(max_shard_rel, rel) + + merged, _ = _merge_partials(shard_o, shard_lse) + merge_rel = (merged - ref_o).abs().max().item() / max( + ref_o.abs().max().item(), 1e-6 + ) + return max_shard_rel, merge_rel + + +# -------------------------------------------------------------------------- +# tests +# -------------------------------------------------------------------------- + + +@requires_cprr +@pytest.mark.parametrize( + "heads_per_rank", + [ + 16, # 128 gathered heads -- a NATIVE cprr kernel, no padding + 12, # 96 gathered heads -- K3's DSpark target, must pad up to 128 + ], + ids=["native-128", "padded-96-to-128"], +) +@torch.inference_mode() +def test_cprr_shards_match_reference_and_merge_to_full_context(heads_per_rank): + """Every DCP shard must be right, and their LSE merge must reproduce + full-context attention. + + A shard that ignored global positions (applying a local causal tail + instead) still produces plausible-looking output; it only diverges from the + reference on the tokens whose causal window is truncated. Checking the merge + alone would hide that, so both are asserted. + """ + shard_rel, merge_rel = _numerics(heads_per_rank) + assert shard_rel < SHARD_RTOL, f"per-shard rel err {shard_rel:.3e}" + assert merge_rel < MERGE_RTOL, f"merged rel err {merge_rel:.3e}" + + +@requires_cprr +@torch.inference_mode() +def test_wrong_cp_rank_breaks_the_result(): + """Positive control: without it, a passing run above proves nothing. + + Feeding shard r's pages to the kernel while telling it that it is rank r+1 + shifts every reconstructed global position by one, so both the causal mask + and the round-robin reconstruction are wrong. If the error does NOT blow up, + the kernel is not using cp_rank and the test above has no power. + """ + shard_rel, merge_rel = _numerics(16, sabotage=True) + assert shard_rel > SHARD_RTOL or merge_rel > MERGE_RTOL, ( + f"sabotaged run still passed (shard {shard_rel:.3e}, merge " + f"{merge_rel:.3e}) -- cp_rank is not reaching the kernel" + ) + + +@requires_cprr +def test_cprr_kernel_is_selected_for_the_k3_shape(): + """The K3 DSpark target shape must actually reach the cprr route. + + Guards against a silent fall-out to segmented/Triton, which is what the + stock configuration does and what this route exists to avoid. + """ + from vllm.v1.attention.backends.mla import rocm_aiter_mla as m + + assert ( + m._asm_dcp_verify_configured( + dcp_world_size=DCP, cp_interleave=1, multi_token_decode=True + ) + is True + ) + assert QLEN >= m._MIN_CPRR_QLEN + assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR + assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS diff --git a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py index 1970dbb54f..309bfd609c 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py +++ b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py @@ -87,21 +87,31 @@ def _builder( kv_cache_dtype: str = "auto", dcp_world_size: int = 1, dcp_rank: int = 0, + asm_dcp_verify: bool = False, + decode_causal: bool = True, + supports_segmented_dcp_verify: bool | None = None, ): + decode_num_heads = num_heads * dcp_world_size + asm_heads = ( + rocm_aiter_mla._asm_dcp_verify_heads(decode_num_heads) if asm_dcp_verify else 0 + ) + if supports_segmented_dcp_verify is None: + supports_segmented_dcp_verify = rocm_aiter_mla._segmented_dcp_verify_supported( + dcp_world_size, 1 + ) stub = SimpleNamespace( device=torch.device("cpu"), num_heads=num_heads, # DCP gathers every rank's head shard before decode, and the routing # predicates read the gathered count. - _decode_num_heads=num_heads * dcp_world_size, + _decode_num_heads=decode_num_heads, dcp_world_size=dcp_world_size, dcp_rank=dcp_rank, cp_kv_cache_interleave_size=1, # Mirrors the production constructor: configuration decides whether the - # segmented route is available, query length decides per batch. - _supports_segmented_dcp_verify=rocm_aiter_mla._segmented_dcp_verify_supported( - dcp_world_size, 1 - ), + # segmented route is available, query length decides per batch. CPRR + # preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back. + _supports_segmented_dcp_verify=supports_segmented_dcp_verify, # Derived once in the real constructor, so derive it once here too. _segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size), _dcp_verify_buffers=None, @@ -122,7 +132,11 @@ def _builder( _uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len), _use_persistent_metadata=False, kernel_block_size=kernel_block_size, - _num_attention_heads=AiterMLAHelper.get_actual_mla_num_heads(num_heads), + _num_attention_heads=( + asm_heads + if asm_dcp_verify + else AiterMLAHelper.get_actual_mla_num_heads(num_heads) + ), _mla_work_meta_data=torch.empty(1, dtype=torch.int32), _mla_work_info_set=torch.empty(1, dtype=torch.int32), _mla_work_indptr=torch.empty(1, dtype=torch.int32), @@ -132,7 +146,19 @@ def _builder( _mla_q_dtype=torch.bfloat16, _mla_kv_dtype=torch.bfloat16, decode_attn_out_dtype=torch.bfloat16, - _decode_causal=True, + _decode_causal=decode_causal, + # Default off so existing cases keep exercising segmented/persistent. + # Pass asm_dcp_verify=True for the CPRR route and its qlen-floor fallback. + _asm_dcp_verify=asm_dcp_verify, + _asm_dcp_verify_heads=asm_heads, + _g_kv_indptr_buf=( + torch.zeros(max_decode_rows + 1, dtype=torch.int32) + if asm_dcp_verify and dcp_world_size > 1 + else None + ), + # Only read when the asm route is on; the real builder derives it from + # the device CU count. + _mla_max_split_per_batch=256, ) # Bound to the stub rather than faked: the verify flatten's per-row view is # part of what _build_decode is being tested for. @@ -287,6 +313,209 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch): assert metadata.dcp_verify is not None +def _cprr_decode_batch(builder, *, qlen: int, num_reqs: int = 1): + seq_lens = torch.full((num_reqs,), 16, dtype=torch.int32) + query_start_loc = torch.arange(0, (num_reqs + 1) * qlen, qlen, dtype=torch.int32) + return AiterMLAMetadataBuilder._build_decode( + builder, + block_table_tensor=torch.zeros(num_reqs, 8, dtype=torch.int32), + seq_lens_device=seq_lens, + max_seq_len=16, + query_start_loc_cpu=query_start_loc, + query_start_loc_device=query_start_loc, + num_decode_tokens=num_reqs * qlen, + max_query_len=qlen, + dcp_tot_seq_lens_device=seq_lens * 8, + ) + + +def test_asm_preferred_qlen2_falls_back_to_segmented(monkeypatch): + """K=4 can clamp to qlen 2; CPRR has no kernel, so use segmented MLA.""" + monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True) + get_mla_metadata_v1 = mock.MagicMock() + monkeypatch.setitem( + sys.modules, + "aiter", + SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1), + ) + + metadata = _cprr_decode_batch( + _builder( + mtp_decode_qlen=5, + dcp_world_size=8, + num_heads=12, + asm_dcp_verify=True, + kernel_block_size=2, + ), + qlen=2, + ) + + assert metadata.dcp_verify is not None + assert metadata.g_kv_indptr is None + assert not metadata.has_persistent_metadata + get_mla_metadata_v1.assert_not_called() + + +def test_asm_preferred_qlen_at_cprr_floor_stays_on_cprr(monkeypatch): + monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True) + get_mla_metadata_v1 = mock.MagicMock() + monkeypatch.setitem( + sys.modules, + "aiter", + SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1), + ) + monkeypatch.setattr( + rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel() + ) + + metadata = _cprr_decode_batch( + _builder( + mtp_decode_qlen=5, + dcp_world_size=8, + num_heads=12, + asm_dcp_verify=True, + ), + qlen=rocm_aiter_mla._MIN_CPRR_QLEN, + ) + + assert metadata.dcp_verify is None + assert metadata.g_kv_indptr is not None + assert metadata.has_persistent_metadata + assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is True + + +def test_cprr_capable_single_token_decode_keeps_persistent_metadata(monkeypatch): + """The CPRR qlen floor must not suppress the ordinary qlen-1 schedule.""" + get_mla_metadata_v1 = mock.MagicMock() + monkeypatch.setitem( + sys.modules, + "aiter", + SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1), + ) + monkeypatch.setattr( + rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel() + ) + + metadata = _cprr_decode_batch( + _builder( + mtp_decode_qlen=5, + dcp_world_size=8, + num_heads=12, + asm_dcp_verify=True, + ), + qlen=1, + ) + + assert metadata.g_kv_indptr is None + assert metadata.has_persistent_metadata + assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None + + +def test_non_causal_dcp_block_uses_plain_mask0_metadata(monkeypatch): + """A non-causal block needs no CPRR global-position window or head pad.""" + get_mla_metadata_v1 = mock.MagicMock() + monkeypatch.setitem( + sys.modules, + "aiter", + SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1), + ) + monkeypatch.setattr( + rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel() + ) + builder = _builder( + mtp_decode_qlen=5, + dcp_world_size=8, + num_heads=12, + asm_dcp_verify=True, + ) + builder._decode_causal = False + + metadata = _cprr_decode_batch(builder, qlen=4) + + assert metadata.dcp_verify is None + assert metadata.g_kv_indptr is None + assert metadata.has_persistent_metadata + assert get_mla_metadata_v1.call_args.args[3] == 96 + assert get_mla_metadata_v1.call_args.args[5] is False + assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None + + +def test_asm_qlen2_without_segmented_fails_during_build(monkeypatch): + monkeypatch.setitem( + sys.modules, + "aiter", + SimpleNamespace(get_mla_metadata_v1=mock.MagicMock()), + ) + monkeypatch.setattr( + rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel() + ) + + with pytest.raises(RuntimeError, match="requires either segmented MLA"): + _cprr_decode_batch( + _builder( + mtp_decode_qlen=5, + dcp_world_size=8, + num_heads=12, + asm_dcp_verify=True, + supports_segmented_dcp_verify=False, + ), + qlen=2, + ) + + +def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch): + """Process-level CPRR capability must not pad or route a non-causal batch.""" + captured = {} + + def fake_aiter_decode(q, kv_buffer, out, *args, **kwargs): + captured["q_heads"] = q.shape[1] + captured["kwargs"] = kwargs + return None, torch.zeros(q.shape[0], q.shape[1]) + + monkeypatch.setattr( + rocm_aiter_mla, "_get_aiter_mla_decode", lambda: fake_aiter_decode + ) + + impl = object.__new__(AiterMLAImpl) + impl.num_heads = 12 + impl.dcp_world_size = 8 + impl.kv_cache_dtype = "auto" + impl.kv_lora_rank = 512 + impl.qk_rope_head_dim = 64 + impl.scale = 576**-0.5 + decode = SimpleNamespace( + max_qo_len=4, + qo_indptr=torch.tensor([0, 4], dtype=torch.int32), + paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32), + paged_kv_indices=torch.zeros(1, dtype=torch.int32), + paged_kv_last_page_len=torch.ones(1, dtype=torch.int32), + use_gluon_decode=False, + use_gluon_verify=False, + dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN, + dcp_verify=None, + g_kv_indptr=None, + has_persistent_metadata=False, + attn_out_dtype=torch.bfloat16, + asm_decode_num_heads=0, + mla_num_kv_splits=0, + cp_world_size=8, + cp_rank=0, + min_kv_seq_len=1, + ) + attn_metadata = SimpleNamespace(decode=decode, causal=False, work_meta_data=None) + layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0)) + q = torch.zeros(4, 96, 576, dtype=torch.bfloat16) + + output, lse = impl.forward_mqa(q, torch.zeros(1, 1, 576), attn_metadata, layer) + + assert captured["q_heads"] == 96 + assert captured["kwargs"]["causal"] is False + assert "g_kv_indptr" not in captured["kwargs"] + assert "num_kv_splits" not in captured["kwargs"] + assert output.shape[1] == 96 + assert lse is not None and lse.shape == (4, 96) + + def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch): """Single-token DCP decode must come back with an LSE the merge can use. @@ -325,9 +554,14 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch): paged_kv_last_page_len=torch.ones(num_tokens, dtype=torch.int32), use_gluon_decode=False, use_gluon_verify=False, + dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN, dcp_verify=None, has_persistent_metadata=False, attn_out_dtype=torch.bfloat16, + # 0 == the round-robin (cprr) asm route is off, this rank's decode takes + # the existing path. Mirrors the AiterMLADecodeMetadata default. + asm_decode_num_heads=0, + g_kv_indptr=None, ) attn_metadata = SimpleNamespace(decode=decode, causal=True, work_meta_data=None) layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0)) @@ -453,6 +687,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch): ), use_gluon_decode=False, use_gluon_verify=False, + dcp_route=rocm_aiter_mla._DCPDecodeRoute.SEGMENTED, dcp_verify=view, attn_out_dtype=torch.bfloat16, ) diff --git a/vllm/envs.py b/vllm/envs.py index f663414a91..5fee7e172e 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -145,6 +145,7 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_USE_AITER_MLA: bool = True VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto" + VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["asm", "segmented"] = "segmented" VLLM_ROCM_USE_AITER_MHA: bool = True VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False @@ -1309,6 +1310,15 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ROCM_USE_AITER_MLA": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_MLA", "True").lower() in ("true", "1") ), + # Kernel for causal multi-token (spec-decode) verify steps under decode + # context parallelism on gfx950: "asm" uses AITER's round-robin ASM + # decode, "segmented" the Triton segmented MLA path. + "VLLM_ROCM_AITER_MLA_DCP_VERIFY": env_with_choices( + "VLLM_ROCM_AITER_MLA_DCP_VERIFY", + "segmented", + ["asm", "segmented"], + case_sensitive=False, + ), # Small-head (<16) AITER MLA decode kernel selection. Small head counts # (e.g. Kimi-K3: 12 heads/rank at TP8, 6 at TP16) can decode either through # the Gluon small-head kernel or through the padded persistent-scheduling diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index 1ab4df0664..f48dedb1d6 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -3,11 +3,13 @@ import functools from dataclasses import dataclass +from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, ClassVar, Final import torch +from vllm import envs from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig from vllm.config.cache import CacheDType @@ -239,6 +241,78 @@ def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) -> ) +# DCP decode routing +# +# With decode context parallelism each rank holds a round-robin shard of the KV +# cache (token t on rank t % dcp_world_size; requires interleave size 1) and +# attends over it with all DCP-gathered query heads. Partial outputs are merged +# across ranks by LSE. Each decode batch takes one route: +# +# * plain: qlen == 1 or a non-causal block. Every query sees every local token. +# * cprr: causal qlen >= _MIN_CPRR_QLEN, gfx950, speculative decoding. AITER's +# round-robin ASM decode rebuilds global positions from g_kv_indptr, cp_rank +# and cp_world_size and applies the causal window in-kernel. Kernels exist only +# at _NATIVE_CPRR_HEADS; other gathered head counts are padded up. +# * segmented: causal qlen > 1 when cprr is not selected or qlen is below +# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV length +# encodes its causal window. +# +# The plain kernel must never serve a causal qlen > 1 block: it applies +# causality on local indices, which is wrong over a round-robin shard. +_NATIVE_CPRR_HEADS: Final = (16, 32, 64, 128) +_MIN_CPRR_QLEN: Final = 3 + + +class _DCPDecodeRoute(Enum): + PLAIN = "plain" + SEGMENTED = "segmented" + CPRR = "cprr" + + +def _asm_dcp_verify_heads(decode_num_heads: int) -> int: + """Head count the cprr asm decode runs at, or 0 if it cannot serve this shape.""" + if decode_num_heads in _NATIVE_CPRR_HEADS: + return decode_num_heads + for nat in _NATIVE_CPRR_HEADS: + if nat >= decode_num_heads: + return nat + return 0 + + +def _select_dcp_decode_route( + supports_segmented: bool, + causal: bool, + max_qo_len: int, + asm_selected: bool, +) -> _DCPDecodeRoute: + """Select one DCP decode route for this batch; see DCP decode routing.""" + if not causal or max_qo_len <= 1: + return _DCPDecodeRoute.PLAIN + if asm_selected and max_qo_len >= _MIN_CPRR_QLEN: + return _DCPDecodeRoute.CPRR + if supports_segmented: + return _DCPDecodeRoute.SEGMENTED + raise RuntimeError( + "ROCM_AITER_MLA DCP multi-token verify requires either segmented " + "MLA or the round-robin asm decode." + ) + + +def _asm_dcp_verify_configured( + dcp_world_size: int, cp_interleave: int, multi_token_decode: bool +) -> bool: + """Whether asm cprr DCP verify is reachable in this process. + + Requires DCP with interleave size 1, speculative decoding, and gfx950 (the + only arch AITER builds cprr kernels for). + """ + if not (dcp_world_size > 1 and cp_interleave == 1 and multi_token_decode): + return False + from vllm.platforms.rocm import on_gfx950 + + return bool(on_gfx950()) + + def _aiter_mla_small_head_mode() -> str: """Small-head (<16) MLA decode kernel selection. @@ -410,8 +484,19 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata): max_qo_len: int | None = None # Minimum KV length used by Gluon to choose a safe split count. min_kv_seq_len: int = 1 + # Exactly one DCP decode route is selected by the metadata builder. + dcp_route: _DCPDecodeRoute = _DCPDecodeRoute.PLAIN # Set exactly when this batch routes to segmented DCP verification. dcp_verify: AiterMLADCPVerifyMetadata | None = None + # cprr only: global per-request page indptr and this rank's position. + g_kv_indptr: torch.Tensor | None = None + cp_world_size: int = 1 + cp_rank: int = 0 + # Padded head count cprr runs at; 0 when this batch is not on cprr. + asm_decode_num_heads: int = 0 + # Must equal the split cap the reduce scratch was sized with, or the kernel + # writes past it and faults. + mla_num_kv_splits: int = 0 # Small-head decode uses Gluon (avoids padding to 16). use_gluon_decode: bool = False # Small-head non-DCP multi-token verify uses Gluon's native MTP entry. @@ -460,6 +545,9 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): # Set from the common metadata every build; a batch is causal unless the # drafter says otherwise. _decode_causal: bool = True + # A non-causal draft needs no cross-shard causal window: the ordinary + # mask0 decode returns per-row LSE for the DCP merge. + supports_non_causal_multi_token_dcp: ClassVar[bool] = True @staticmethod def _uniform_padded_mtp_qo_len( @@ -513,15 +601,48 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): parallel_config.decode_context_parallel_size, parallel_config.cp_kv_cache_interleave_size, ) + asm_dcp_verify_config = ( + _asm_dcp_verify_configured( + parallel_config.decode_context_parallel_size, + parallel_config.cp_kv_cache_interleave_size, + multi_token_decode=vllm_config.speculative_config is not None, + ) + and envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "asm" + ) super().__init__( kv_cache_spec, layer_names, vllm_config, device, AiterMLAMetadata, - supports_dcp_with_varlen=supports_segmented_dcp_verify, + supports_dcp_with_varlen=( + supports_segmented_dcp_verify or asm_dcp_verify_config + ), ) + self._asm_dcp_verify = False + self._asm_dcp_verify_heads = 0 + # The head-count half of the route decision needs self.num_heads, so only + # the config half is known before super().__init__(). + if asm_dcp_verify_config: + self._asm_dcp_verify_heads = _asm_dcp_verify_heads( + self.num_heads * self.dcp_world_size + ) + if not self._asm_dcp_verify_heads: + raise ValueError( + "VLLM_ROCM_AITER_MLA_DCP_VERIFY=asm, but the round-robin asm " + f"decode has no kernel for {self.num_heads * self.dcp_world_size} " + f"DCP-gathered heads (native counts: {_NATIVE_CPRR_HEADS}). " + "Set VLLM_ROCM_AITER_MLA_DCP_VERIFY=segmented." + ) + self._asm_dcp_verify = True self._supports_segmented_dcp_verify = supports_segmented_dcp_verify + self._mla_max_split_per_batch = 0 + if self._asm_dcp_verify: + # Cap on KV splits per batch; at low batch the KV axis is the only + # parallelism. Buffer sizing and every runtime call must see this value. + self._mla_max_split_per_batch = torch.cuda.get_device_properties( + device + ).multi_processor_count self.compilation_config = vllm_config.compilation_config self.decode_attn_out_dtype = vllm_config.model_config.dtype @@ -576,6 +697,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads( self._decode_num_heads ) + # the cprr kernel exists only at the native head + # counts, so run (and size the persistent schedule) at the padded count. + # 96 is 16-aligned, so get_actual_mla_num_heads leaves it alone and the + # pad has to be applied here explicitly. + if self._asm_dcp_verify: + self._num_attention_heads = self._asm_dcp_verify_heads kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto") if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"): kv_cache_dtype_str = "fp8" @@ -619,6 +746,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): kv_dtype, is_sparse=False, fast_mode=True, + # Must match max_split_per_batch in get_mla_metadata_v1; without it + # aiter sizes the reduce scratch from capacity (~9 GiB). + **( + dict(max_split_per_batch=self._mla_max_split_per_batch) + if self._asm_dcp_verify + else {} + ), ) self._mla_work_meta_data = torch.empty( work_meta_data_size, dtype=work_meta_data_type, device=device @@ -664,6 +798,14 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): device, ) + # Full cudagraphs need a stable address for the cprr global page indptr. + # Zero-initialized; element 0 is the indptr base and is never rewritten. + self._g_kv_indptr_buf: torch.Tensor | None = None + if self._asm_dcp_verify: + self._g_kv_indptr_buf = torch.zeros( + max_num_reqs + 1, dtype=torch.int32, device=device + ) + # Persistent buffers for segmented DCP verification. Captured graphs # require stable addresses while row lengths vary between replays. self._dcp_verify_buffers: AiterMLADCPVerifyMetadata | None = None @@ -683,8 +825,11 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): ) if self._supports_segmented_dcp_verify and self._mtp_decode_qlen > 1: - # A DCP rank's shard of the longest sequence bounds every verify - # row, and full graphs need that bound to be constant. + # Allocate even when CPRR is the preferred route: a later + # replay can fall below _MIN_CPRR_QLEN and needs these + # addresses. A DCP rank's shard of the longest sequence + # bounds every verify row, and full graphs need that bound + # to be constant. num_dcp_partitions = ( self.dcp_world_size * self.cp_kv_cache_interleave_size ) @@ -1063,6 +1208,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): pad_uniform_mtp = padded_mtp_qo_len > 0 seq_lens_for_kernel = seq_lens_device + # the global lengths need the same dummy-row + # treatment as the local ones below, but only for g_kv_indptr -- the + # metadata field keeps the true values, which is what the segmented + # route wants (it derives its own row lens and gives a dummy row + # length 0, correctly). + g_tot_seq_lens = dcp_tot_seq_lens_device num_kernel_reqs = num_reqs if pad_uniform_mtp: qo_lens_device = ( @@ -1074,6 +1225,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): seq_lens_for_kernel, seq_lens_for_kernel.new_full((), max_qo_len), ) + # A dummy row just got a LOCAL length of max_qo_len, so its GLOBAL + # length has to say the same thing or the cprr mask is degenerate: + # global_len 0 puts the row query positions at {-3,-2,-1} while its + # local shard sits at {rank, W+rank, 2W+rank}, so every entry is + # masked and the row softmaxes over nothing. max_qo_len * W is + # exactly the global length whose round-robin shard is max_qo_len + # on every rank. + if g_tot_seq_lens is not None: + g_tot_seq_lens = torch.where( + qo_lens_device > 0, + g_tot_seq_lens, + g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size), + ) if self._graph_seq_lens is not None: self._graph_seq_lens[:num_kernel_reqs].copy_( @@ -1106,16 +1270,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): causal, self._kv_cache_bytes, ) - use_segmented_dcp_verify = ( - self._supports_segmented_dcp_verify and max_qo_len > 1 - ) + dcp_route = _DCPDecodeRoute.PLAIN + if self.dcp_world_size > 1: + dcp_route = _select_dcp_decode_route( + self._supports_segmented_dcp_verify, + causal, + int(max_qo_len), + self._asm_dcp_verify, + ) # Segmented DCP verify carries its own per-row subpage table, so the # flat per-token view is dead work for it. Leave the buffer alone and # hand the metadata None, so a future reader cannot pick up whatever # the previous batch left behind. paged_kv_indices = None - if not use_segmented_dcp_verify: + if dcp_route is not _DCPDecodeRoute.SEGMENTED: if self.compilation_config.cudagraph_mode.has_full_cudagraphs(): self.paged_kv_indices.fill_(-1) @@ -1169,6 +1338,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): else: qo_indptr = query_start_loc_device[: 1 + num_kernel_reqs] + g_kv_indptr = None + if dcp_route is _DCPDecodeRoute.CPRR and g_tot_seq_lens is not None: + assert self._g_kv_indptr_buf is not None + _ngk = g_tot_seq_lens.shape[0] + g_kv_indptr = self._g_kv_indptr_buf[: _ngk + 1] + g_kv_indptr[1:].copy_(g_tot_seq_lens.cumsum(dim=0, dtype=torch.int32)) + # A FULL cudagraph replays at its capture-time request count, so a + # smaller real batch leaves the tail holding the PREVIOUS step's + # cumsum. Flatten it the way paged_kv_indptr is flattened below, so + # padding rows read global_len == 0 and agree with their zero-length + # local shard instead of a stale global length. + self._g_kv_indptr_buf[_ngk + 1 :].fill_(g_kv_indptr[-1]) + has_persistent_metadata = False # Only the asm decode consumes the schedule, so gate on the routing # rather than on num_heads >= 16, which denies it to a padded rank @@ -1178,7 +1360,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): use_persistent_metadata = ( not use_gluon_decode and not use_gluon_verify - and not use_segmented_dcp_verify + and dcp_route is not _DCPDecodeRoute.SEGMENTED # A padded rank has no bf16 persistent kernel past qlen 4 where the # gfx950 fold is absent; the non-persistent entry covers it. fp8 # keeps the schedule -- its fold rejects non-persistent outright. @@ -1218,13 +1400,28 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): uni_qo_len = ( max_qo_len if pad_uniform_mtp or torch.all(qo_len == max_qo_len) else -1 ) + cprr_kwargs: dict = {} + is_causal = causal + if self._asm_dcp_verify: + cprr_kwargs = dict( + max_split_per_batch=self._mla_max_split_per_batch, + intra_batch_mode=False, + ) + if dcp_route is _DCPDecodeRoute.CPRR: + is_causal = False + cprr_kwargs["is_cp_round_robin"] = True + metadata_num_heads = ( + self._asm_dcp_verify_heads + if dcp_route is _DCPDecodeRoute.CPRR + else AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads) + ) get_mla_metadata_v1( qo_indptr, paged_kv_indptr, paged_kv_last_page_len, - self._num_attention_heads, + metadata_num_heads, 1, - causal, + is_causal, self._mla_work_meta_data, self._mla_work_info_set, self._mla_work_indptr, @@ -1238,6 +1435,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): fast_mode=True, dtype_q=self._mla_q_dtype, dtype_kv=self._mla_kv_dtype, + **cprr_kwargs, ) has_persistent_metadata = True @@ -1261,7 +1459,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): min_kv_seq_len = int(per_req_len.min().item()) dcp_verify = None - if use_segmented_dcp_verify: + if dcp_route is _DCPDecodeRoute.SEGMENTED: assert dcp_tot_seq_lens_device is not None dcp_verify = self._build_dcp_verify_row_view( int(max_qo_len), @@ -1279,7 +1477,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): dcp_tot_seq_lens=dcp_tot_seq_lens_device, max_qo_len=max_qo_len, min_kv_seq_len=min_kv_seq_len, + dcp_route=dcp_route, dcp_verify=dcp_verify, + g_kv_indptr=g_kv_indptr, + cp_world_size=self.dcp_world_size, + cp_rank=self.dcp_rank, + asm_decode_num_heads=( + self._asm_dcp_verify_heads if dcp_route is _DCPDecodeRoute.CPRR else 0 + ), + mla_num_kv_splits=( + self._mla_max_split_per_batch + if dcp_route is _DCPDecodeRoute.CPRR + else 0 + ), use_gluon_decode=use_gluon_decode, use_gluon_verify=use_gluon_verify, attn_out_dtype=self.decode_attn_out_dtype, @@ -2062,8 +2272,10 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): ) return o, None - verify = decode.dcp_verify - if verify is not None: + dcp_route = decode.dcp_route + if dcp_route is _DCPDecodeRoute.SEGMENTED: + verify = decode.dcp_verify + assert verify is not None if type(q) is tuple: q_nope, q_pe = q else: @@ -2085,11 +2297,6 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): decode.attn_out_dtype, ) - if self.dcp_world_size > 1 and int(decode.max_qo_len) > 1: - raise RuntimeError( - "ROCM_AITER_MLA DCP multi-token verify requires segmented MLA." - ) - if type(q) is tuple: q = torch.cat(q, dim=-1) @@ -2102,8 +2309,19 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): "ROCM_AITER_MLA decode expected the DCP-gathered query head count " f"{self._decode_num_heads}, got {q.shape[1]}" ) - mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q) - mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads) + # cprr has kernels only at native head counts, so pad q up (e.g. 96 -> + # 128). Heads are independent in MLA; the extras are sliced off below. + asm_dcp_heads = decode.asm_decode_num_heads + if asm_dcp_heads > self._decode_num_heads: + mla_num_heads = asm_dcp_heads + mla_padded_q = AiterMLAHelper.get_mla_padded_q( + self._decode_num_heads, q, target_heads=mla_num_heads + ) + else: + mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q) + mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads( + self._decode_num_heads + ) o = torch.empty( B, mla_num_heads, @@ -2137,6 +2355,16 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): lse = None if self.dcp_world_size > 1: + if dcp_route is _DCPDecodeRoute.CPRR: + # cprr only: global positions for the in-kernel causal window. + assert decode.g_kv_indptr is not None + mla_kwargs["g_kv_indptr"] = decode.g_kv_indptr + mla_kwargs["cp_world_size"] = decode.cp_world_size + mla_kwargs["cp_rank"] = decode.cp_rank + # Must match the schedule get_mla_metadata_v1 was built with. + assert decode.mla_num_kv_splits > 0 + mla_kwargs["num_kv_splits"] = decode.mla_num_kv_splits + mla_kwargs["intra_batch_mode"] = False # The vLLM custom-op wrapper exposes only the in-place output and # drops aiter's final LSE, which the cross-shard merge needs, so go # through aiter's native entry point on the DCP path. @@ -2151,6 +2379,7 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): decode.max_qo_len, sm_scale=self.scale, return_lse=True, + causal=attn_metadata.causal, **mla_kwargs, ) assert lse is not None, ( @@ -2172,7 +2401,13 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): **mla_kwargs, ) - output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o) - if lse is not None: - lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse) + if asm_dcp_heads > self._decode_num_heads: + # Keep a view: a post-kernel copy races the symm-mem a2a combine. + output = o[:, : self._decode_num_heads] + if lse is not None: + lse = lse[:, : self._decode_num_heads, ...] + else: + output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o) + if lse is not None: + lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse) return output, lse