[ROCm][MLA] Add an AITER ASM round-robin decode route for DCP multi-token verify (#56861)
Signed-off-by: Xiaohu Guo <Xiaohu.Guo@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: seungrokj <144636725+seungrokj@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Claude Opus 5
seungrokj
Cursor
parent
5ba2e3090b
commit
25624f65cf
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Routing/selection tests for the AITER ASM cprr DCP-verify route.
|
||||
|
||||
Companion to the upstream segmented-verify tests. Everything here is pure CPU:
|
||||
it pins the *decisions* (which route a KV group takes, which head count the
|
||||
kernel runs at, which configurations are refused) rather than kernel numerics.
|
||||
The numerics parity test against the segmented route needs a GPU and both
|
||||
routes built; it is marked and skipped when unavailable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
rocm_aiter_mla = pytest.importorskip(
|
||||
"vllm.v1.attention.backends.mla.rocm_aiter_mla",
|
||||
reason="ROCm AITER MLA backend not available",
|
||||
)
|
||||
|
||||
_heads = rocm_aiter_mla._asm_dcp_verify_heads
|
||||
_configured = rocm_aiter_mla._asm_dcp_verify_configured
|
||||
_select_route = rocm_aiter_mla._select_dcp_decode_route
|
||||
Route = rocm_aiter_mla._DCPDecodeRoute
|
||||
NATIVE = rocm_aiter_mla._NATIVE_CPRR_HEADS
|
||||
MIN_QLEN = rocm_aiter_mla._MIN_CPRR_QLEN
|
||||
|
||||
|
||||
def test_dcp_verify_env_defaults_to_segmented(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", raising=False)
|
||||
assert rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "segmented"
|
||||
|
||||
|
||||
def test_dcp_verify_env_rejects_unknown_route(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "unknown")
|
||||
with pytest.raises(ValueError, match="Invalid value"):
|
||||
_ = rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# _asm_dcp_verify_heads -- which head count the cprr kernel runs at
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", NATIVE)
|
||||
def test_native_head_counts_are_used_as_is(n):
|
||||
assert _heads(n) == n
|
||||
|
||||
|
||||
def test_non_native_head_count_pads_up_to_the_next_native():
|
||||
"""K3 at TP8/DCP8 gathers 96 heads for the target; the kernel has no 96
|
||||
variant, so it must run at 128, not silently fall out of the asm path."""
|
||||
assert _heads(96) == 128
|
||||
assert _heads(1) == 16
|
||||
assert _heads(17) == 32
|
||||
assert _heads(65) == 128
|
||||
|
||||
|
||||
def test_head_count_above_the_largest_native_is_unservable():
|
||||
"""0 means 'cannot serve'; anything else would route a shape the kernel
|
||||
has no build for."""
|
||||
assert _heads(max(NATIVE) + 1) == 0
|
||||
assert _heads(256) == 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# _asm_dcp_verify_configured -- reachability
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def on_gfx950(monkeypatch):
|
||||
"""The cprr kernels are built only for gfx950, so the gate checks the arch.
|
||||
Patched rather than detected, so both branches run on any runner."""
|
||||
import vllm.platforms.rocm as rocm
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx950", lambda: True)
|
||||
|
||||
|
||||
def test_route_needs_dcp(on_gfx950):
|
||||
assert _configured(dcp_world_size=1, cp_interleave=1, multi_token_decode=True) is (
|
||||
False
|
||||
)
|
||||
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is (
|
||||
True
|
||||
)
|
||||
|
||||
|
||||
def test_round_robin_interleave_other_than_one_is_excluded(on_gfx950):
|
||||
"""The kernel's global-position causal window assumes interleave 1, the
|
||||
same restriction the segmented gate carries."""
|
||||
assert _configured(dcp_world_size=8, cp_interleave=2, multi_token_decode=True) is (
|
||||
False
|
||||
)
|
||||
assert _configured(dcp_world_size=8, cp_interleave=16, multi_token_decode=True) is (
|
||||
False
|
||||
)
|
||||
|
||||
|
||||
def test_route_needs_multi_token_decode(on_gfx950):
|
||||
"""Single-token decode is already served. Enabling the route for it would
|
||||
pad the head count on every step and buy nothing."""
|
||||
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=False) is (
|
||||
False
|
||||
)
|
||||
|
||||
|
||||
def test_route_needs_gfx950(monkeypatch):
|
||||
"""hsa/gfx942/mla and hsa/gfx1250/mla carry no cprr rows, so off gfx950 the
|
||||
kernel lookup fails at the first verify step rather than falling back."""
|
||||
import vllm.platforms.rocm as rocm
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx950", lambda: False)
|
||||
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is (
|
||||
False
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# qlen floor
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_min_cprr_qlen_is_above_two():
|
||||
"""Qlen 2 is below CPRR's floor and must use segmented verification."""
|
||||
assert MIN_QLEN > 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"supports,causal,qlen,asm,expected",
|
||||
[
|
||||
(True, True, 1, True, Route.PLAIN),
|
||||
(True, True, 2, True, Route.SEGMENTED),
|
||||
(True, True, 3, True, Route.CPRR),
|
||||
(True, True, 4, True, Route.CPRR),
|
||||
(True, True, 4, False, Route.SEGMENTED),
|
||||
(True, False, 2, True, Route.PLAIN),
|
||||
(True, False, 4, False, Route.PLAIN),
|
||||
(False, True, 3, True, Route.CPRR),
|
||||
],
|
||||
)
|
||||
def test_dcp_decode_route(supports, causal, qlen, asm, expected):
|
||||
assert _select_route(supports, causal, qlen, asm) is expected
|
||||
|
||||
|
||||
def test_causal_multi_token_batch_without_a_valid_route_fails():
|
||||
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
|
||||
_select_route(False, True, 2, True)
|
||||
|
||||
|
||||
# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs
|
||||
# the real builder and the real asm kernel on a gfx950 GPU.
|
||||
@@ -0,0 +1,540 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GPU numerics for the AITER ASM cprr DCP-verify route.
|
||||
|
||||
Runs the REAL metadata builder (``AiterMLAMetadataBuilder.build``, so
|
||||
``get_mla_metadata_v1`` is exercised with the cprr kwargs and the persistent
|
||||
schedule buffers are produced) and the REAL asm kernel, on ONE GPU with the DCP
|
||||
ranks simulated one after another.
|
||||
|
||||
That is legitimate because the aiter MLA backend contains no collective on this
|
||||
path: the query all-gather happens in the *layer*, strictly before
|
||||
``forward_mqa``, so the head fold, the round-robin kernel call and the un-fold
|
||||
are pure per-rank local arithmetic in which ``cp_world_size``/``cp_rank`` are
|
||||
just numbers feeding the causal-mask maths. Restricting a shard to a residue
|
||||
class of global positions is exactly what DCP rank r sees.
|
||||
|
||||
Three properties are checked:
|
||||
* each shard matches an exact torch reference over the positions it holds,
|
||||
with causality applied on the GLOBAL position (the thing cprr reconstructs
|
||||
in-kernel and the segmented route instead carries in per-row lengths);
|
||||
* the LSE merge of all shards reproduces full-context attention;
|
||||
* a shard told the wrong ``cp_rank`` fails both -- without this positive
|
||||
control a passing run proves nothing, since a kernel that ignored global
|
||||
positions still produces plausible-looking output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm AITER MLA tests", allow_module_level=True)
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found # noqa: E402
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla # noqa: E402
|
||||
|
||||
KV_LORA_RANK = 512
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5
|
||||
|
||||
DCP = 8
|
||||
NUM_REQS = 8
|
||||
CTX = 256
|
||||
QLEN = 5 # 1 + num_speculative_tokens; must clear _MIN_CPRR_QLEN
|
||||
|
||||
# fp8-e4m3 keeps 3 mantissa bits, so the relative error floor is ~6.2e-2. Gates
|
||||
# are relative for the same reason: an absolute bound would just be a bound on
|
||||
# the output magnitude.
|
||||
SHARD_RTOL = 1.0e-1
|
||||
MERGE_RTOL = 5.0e-2
|
||||
|
||||
MODEL = "deepseek-ai/DeepSeek-R1"
|
||||
|
||||
|
||||
def _gpu_available() -> bool:
|
||||
return is_aiter_found() and torch.accelerator.is_available()
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not is_aiter_found():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
requires_cprr = pytest.mark.skipif(
|
||||
not (_gpu_available() and _on_gfx950()),
|
||||
reason="cprr DCP verify needs ROCm + AITER on gfx950",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def select_asm_route(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "asm")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# reference maths
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _torch_reference(q_f32, kv_f32, ctx, qlen, scale, positions):
|
||||
"""Exact absorbed-MLA decode over an explicit set of GLOBAL KV positions.
|
||||
|
||||
A shard sees only ``positions``, but causality still uses the global
|
||||
position: query token i (global position ctx + i) may attend to any p in
|
||||
positions with p <= ctx + i. Restricting ``positions`` to a round-robin
|
||||
residue class is precisely DCP rank r's view.
|
||||
"""
|
||||
nr, _, heads, _ = q_f32.shape
|
||||
dev = q_f32.device
|
||||
o = torch.zeros(nr, qlen, heads, KV_LORA_RANK, dtype=torch.float32, device=dev)
|
||||
lse = torch.full((nr, qlen, heads), float("-inf"), dtype=torch.float32, device=dev)
|
||||
for r in range(nr):
|
||||
kv_sel = kv_f32[r].index_select(0, positions)
|
||||
for i in range(qlen):
|
||||
valid = positions <= (ctx + i)
|
||||
if not bool(valid.any()):
|
||||
continue
|
||||
k = kv_sel[valid]
|
||||
s = (q_f32[r, i] @ k.T) * scale
|
||||
m = s.max(dim=-1, keepdim=True).values
|
||||
p = torch.exp(s - m)
|
||||
denom = p.sum(dim=-1, keepdim=True)
|
||||
o[r, i] = (p / denom) @ k[:, :KV_LORA_RANK]
|
||||
lse[r, i] = m.squeeze(-1) + torch.log(denom.squeeze(-1))
|
||||
return o, lse
|
||||
|
||||
|
||||
def _merge_partials(os_, lses):
|
||||
"""Cross-shard softmax merge (natural-log LSE)."""
|
||||
stack_lse = torch.stack(lses)
|
||||
m = stack_lse.max(dim=0).values
|
||||
w = torch.exp(stack_lse - m)
|
||||
num = (w.unsqueeze(-1) * torch.stack(os_)).sum(dim=0)
|
||||
den = w.sum(dim=0).unsqueeze(-1)
|
||||
return num / den, m + torch.log(w.sum(dim=0))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# single-process rank simulation
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeGroup:
|
||||
def __init__(self, world_size: int, rank: int):
|
||||
self.world_size = world_size
|
||||
self.rank_in_group = rank
|
||||
self.rank = rank
|
||||
self.local_rank = rank
|
||||
|
||||
def all_gather(self, tensor, dim=0):
|
||||
raise RuntimeError(
|
||||
"collective called in a single-process test -- the path under test "
|
||||
"is supposed to be collective-free; this indicates a real change"
|
||||
)
|
||||
|
||||
|
||||
def _install_fake_groups(dcp_size: int, cp_rank: int, tp_size: int = 1) -> None:
|
||||
"""Point vLLM's parallel_state at rank-simulating stubs (no NCCL)."""
|
||||
import vllm.distributed.parallel_state as ps
|
||||
import vllm.model_executor.layers.attention.mla_attention as mla_attention
|
||||
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
|
||||
|
||||
dcp = _FakeGroup(dcp_size, cp_rank)
|
||||
tp = _FakeGroup(tp_size, cp_rank)
|
||||
ps.get_dcp_group = lambda: dcp
|
||||
ps.get_tp_group = lambda: tp
|
||||
# Both of these imported the symbol directly at module import time, so
|
||||
# patching parallel_state alone leaves them resolving the real group (and
|
||||
# the builder then reports dcp_world_size 1, silently skipping the route).
|
||||
backend.get_dcp_group = lambda: dcp
|
||||
mla_attention.get_dcp_group = lambda: dcp
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _restore_parallel_groups():
|
||||
"""Undo _install_fake_groups after every test.
|
||||
|
||||
It patches module globals in three modules, so without this the fake DCP
|
||||
group leaks into the rest of the pytest session and silently breaks
|
||||
unrelated tests that expect a real (or absent) group.
|
||||
"""
|
||||
import vllm.distributed.parallel_state as ps
|
||||
import vllm.model_executor.layers.attention.mla_attention as mla_attention
|
||||
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
|
||||
|
||||
saved = [
|
||||
(mod, name, getattr(mod, name))
|
||||
for mod, name in (
|
||||
(ps, "get_dcp_group"),
|
||||
(ps, "get_tp_group"),
|
||||
(backend, "get_dcp_group"),
|
||||
(mla_attention, "get_dcp_group"),
|
||||
)
|
||||
]
|
||||
yield
|
||||
for mod, name, value in saved:
|
||||
setattr(mod, name, value)
|
||||
|
||||
|
||||
class _PrefillBackendStub:
|
||||
"""The base builder does ``attention_layer.prefill_backend.clone()``.
|
||||
|
||||
prefill_backend is consulted only from the chunked-context prefill path,
|
||||
which a decode-only test never enters.
|
||||
"""
|
||||
|
||||
def clone(self):
|
||||
return self
|
||||
|
||||
|
||||
def _make_ctx_layer(vllm_config):
|
||||
"""Stand-in for the registered attention layer in static_forward_context.
|
||||
|
||||
``MLACommonMetadataBuilder.__init__`` reads the MLA dims, ``dcp_manager``
|
||||
and ``prefill_backend`` off this object. ``dcp_manager`` is the Direct-DCP
|
||||
symmetric-memory workspace, used ONLY by the chunked-prefill KV all-gather;
|
||||
it is constructed without running __init__ so no peer pointers are exchanged
|
||||
and no collective is issued. That is what keeps this test single-process.
|
||||
"""
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
|
||||
mgr = object.__new__(MLADCPManager)
|
||||
mgr.init_kv_gather = lambda *a, **k: None
|
||||
|
||||
return SimpleNamespace(
|
||||
non_causal_multi_token_decode=False,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
v_head_dim=QK_NOPE_HEAD_DIM,
|
||||
dcp_manager=mgr,
|
||||
prefill_backend=_PrefillBackendStub(),
|
||||
)
|
||||
|
||||
|
||||
def _ensure_workspace_manager(device: torch.device) -> None:
|
||||
"""Init the global workspace manager, which GPUModelRunner does in production.
|
||||
|
||||
``AiterMLAMetadataBuilder.__init__`` reserves the fp8 prefill
|
||||
persistent-scheduling scratch through ``current_workspace_manager()``, so
|
||||
merely constructing a builder outside a serve trips its assert. Idempotent:
|
||||
one builder is constructed per DCP shard. Deliberately does NOT lock the
|
||||
workspace -- production locks only after warmup, and this keeps allocating
|
||||
across shards.
|
||||
"""
|
||||
from vllm.v1.worker.workspace import (
|
||||
init_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
)
|
||||
|
||||
if not is_workspace_manager_initialized():
|
||||
init_workspace_manager(device)
|
||||
|
||||
|
||||
class _Layer:
|
||||
def __init__(self, device):
|
||||
self._q_scale = torch.tensor(1.0, device=device)
|
||||
self._k_scale = torch.tensor(1.0, device=device)
|
||||
self._v_scale = torch.tensor(1.0, device=device)
|
||||
|
||||
|
||||
def _build_impl(num_heads: int, dcp: int, cp_rank: int):
|
||||
"""Real AiterMLAImpl.forward_mqa bound to a minimally-populated instance.
|
||||
|
||||
Constructing the full impl would drag in kv_b_proj and the rest of the
|
||||
weight-dependent MLA setup -- i.e. model loading, which this test exists to
|
||||
avoid. forward_mqa's fold path reads only the attributes set below.
|
||||
"""
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.kv_lora_rank = KV_LORA_RANK
|
||||
impl.qk_rope_head_dim = QK_ROPE_HEAD_DIM
|
||||
impl.num_heads = num_heads
|
||||
impl.dcp_world_size = dcp
|
||||
impl.dcp_rank = cp_rank
|
||||
impl.pcp_world_size = 1
|
||||
impl.kv_cache_dtype = "fp8"
|
||||
impl.scale = SCALE
|
||||
# `_decode_num_heads` is a read-only property deriving num_heads *
|
||||
# dcp_world_size, both set above; assert rather than assign.
|
||||
assert impl._decode_num_heads == num_heads * dcp
|
||||
return impl
|
||||
|
||||
|
||||
def _make_vllm_config(heads_per_rank: int):
|
||||
"""A DCP8 MLA config with no weights and no distributed init.
|
||||
|
||||
DeepSeek-R1's config supplies the MLA dims (kv_lora_rank 512, rope 64,
|
||||
nope 128) that the aiter asm kernel is built for; only the head count is
|
||||
overridden, to place the GATHERED count (heads_per_rank * DCP) on the
|
||||
kernel variant under test. ``get_num_attention_heads`` reads
|
||||
model_arch_config, a snapshot taken at config-creation time, so patching
|
||||
hf_config alone would be inert.
|
||||
"""
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name=MODEL,
|
||||
tensor_parallel_size=1,
|
||||
max_model_len=2048,
|
||||
max_num_seqs=NUM_REQS * 2,
|
||||
max_num_batched_tokens=4096,
|
||||
)
|
||||
vllm_config.model_config.model_arch_config.total_num_attention_heads = (
|
||||
heads_per_rank
|
||||
)
|
||||
# Without a speculative config the batch reorder threshold stays at 1 and a
|
||||
# qlen-QLEN row is classified as a PREFILL, so the decode path under test is
|
||||
# never reached. _init_reorder_batch_threshold reads exactly these two
|
||||
# fields, so a stand-in is enough and avoids pulling in a draft model.
|
||||
vllm_config.speculative_config = SimpleNamespace(
|
||||
num_speculative_tokens=QLEN - 1, parallel_drafting=False
|
||||
)
|
||||
vllm_config.cache_config.cache_dtype = "fp8"
|
||||
vllm_config.parallel_config.decode_context_parallel_size = DCP
|
||||
vllm_config.parallel_config.cp_kv_cache_interleave_size = 1
|
||||
return vllm_config
|
||||
|
||||
|
||||
def _run_shard(
|
||||
vllm_config,
|
||||
spec,
|
||||
layer_name,
|
||||
device,
|
||||
cp_rank,
|
||||
kern_rank,
|
||||
q_flat,
|
||||
kv_flat,
|
||||
nr,
|
||||
qlen,
|
||||
ctx,
|
||||
dcp,
|
||||
pos,
|
||||
):
|
||||
"""Build real metadata for one DCP rank and run the real asm kernel.
|
||||
|
||||
``kern_rank`` is separate from ``cp_rank`` so the positive control can hand
|
||||
rank r's pages to the kernel while telling it that it is rank r+1.
|
||||
"""
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
|
||||
|
||||
L = ctx + qlen
|
||||
local_len = int(pos.numel())
|
||||
|
||||
# One GLOBAL kv cache: page (req*L + p) holds request req position p. Shards
|
||||
# select pages by residue class, so every shard reads the same underlying
|
||||
# bytes and no re-quantisation noise is introduced between arms.
|
||||
block_table = (
|
||||
torch.arange(nr, device=device).view(nr, 1) * L + pos.view(1, -1)
|
||||
).to(torch.int32)
|
||||
|
||||
# seq_lens is the GLOBAL context; dcp_local_seq_lens is what this rank
|
||||
# actually holds. Passing the global length as the local one walks the
|
||||
# kernel off the end of the page table.
|
||||
seq_lens_cpu = torch.full((nr,), L, dtype=torch.int32)
|
||||
local_cpu = torch.full((nr,), local_len, dtype=torch.int32)
|
||||
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=torch.arange(
|
||||
0, nr * qlen + 1, qlen, dtype=torch.int32, device=device
|
||||
),
|
||||
query_start_loc_cpu=torch.arange(0, nr * qlen + 1, qlen, dtype=torch.int32),
|
||||
seq_lens=seq_lens_cpu.to(device),
|
||||
num_reqs=nr,
|
||||
num_actual_tokens=nr * qlen,
|
||||
max_query_len=qlen,
|
||||
max_seq_len=L,
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=torch.arange(nr * qlen, dtype=torch.int64, device=device),
|
||||
causal=True,
|
||||
dcp_local_seq_lens=local_cpu.to(device),
|
||||
dcp_local_seq_lens_cpu_upper_bound=local_cpu,
|
||||
)
|
||||
|
||||
_install_fake_groups(dcp, kern_rank)
|
||||
builder = AiterMLAMetadataBuilder(spec, [layer_name], vllm_config, device)
|
||||
md = builder.build(0, common)
|
||||
|
||||
impl = _build_impl(builder.num_heads, dcp, kern_rank)
|
||||
o, lse = impl.forward_mqa(q_flat, kv_flat, md, _Layer(device))
|
||||
torch.accelerator.synchronize()
|
||||
return md, o, lse, local_len
|
||||
|
||||
|
||||
def _numerics(heads_per_rank: int, sabotage: bool = False):
|
||||
"""Run every DCP shard and return (per-shard max rel err, merged rel err)."""
|
||||
import aiter
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
|
||||
device = torch.device("cuda", 0)
|
||||
torch.accelerator.set_device_index(device.index)
|
||||
torch.manual_seed(0)
|
||||
_ensure_workspace_manager(device)
|
||||
|
||||
_install_fake_groups(DCP, 0)
|
||||
vllm_config = _make_vllm_config(heads_per_rank)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
|
||||
layer_name = "model.layers.0.self_attn.attn"
|
||||
vllm_config.compilation_config.static_forward_context[layer_name] = (
|
||||
_make_ctx_layer(vllm_config)
|
||||
)
|
||||
# block_size 1: one page per token, so page ids are positions and the
|
||||
# residue-class page table is expressible directly.
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=1,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
non_causal_multi_token_decode=False,
|
||||
)
|
||||
|
||||
nr, ctx, qlen = NUM_REQS, CTX, QLEN
|
||||
L = ctx + qlen
|
||||
gathered_heads = heads_per_rank * DCP
|
||||
kv_dtype = aiter.dtypes.fp8
|
||||
|
||||
kv_flat = torch.randn(
|
||||
nr * L, 1, HEAD_SIZE, dtype=torch.float32, device=device
|
||||
).to(kv_dtype)
|
||||
q_flat = (
|
||||
torch.randn(
|
||||
nr * qlen,
|
||||
gathered_heads,
|
||||
HEAD_SIZE,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
.mul_(0.5)
|
||||
.to(kv_dtype)
|
||||
)
|
||||
# Dequantise for the reference so the comparison measures the kernel,
|
||||
# not the fp8 cast that both sides share.
|
||||
kv_f32 = kv_flat.float().view(nr, L, HEAD_SIZE)
|
||||
q_f32 = q_flat.float().view(nr, qlen, gathered_heads, HEAD_SIZE)
|
||||
|
||||
ref_o, _ = _torch_reference(
|
||||
q_f32, kv_f32, ctx, qlen, SCALE, torch.arange(L, device=device)
|
||||
)
|
||||
|
||||
shard_o, shard_lse = [], []
|
||||
max_shard_rel = 0.0
|
||||
for r in range(DCP):
|
||||
pos = torch.arange(r, L, DCP, device=device) # residue class r
|
||||
kern_rank = (r + 1) % DCP if sabotage else r
|
||||
md, o, lse, _ = _run_shard(
|
||||
vllm_config,
|
||||
spec,
|
||||
layer_name,
|
||||
device,
|
||||
r,
|
||||
kern_rank,
|
||||
q_flat,
|
||||
kv_flat,
|
||||
nr,
|
||||
qlen,
|
||||
ctx,
|
||||
DCP,
|
||||
pos,
|
||||
)
|
||||
# The route actually under test must have been taken; a silent
|
||||
# fall-out to Triton/segmented would otherwise pass quietly.
|
||||
assert md.decode.dcp_route is rocm_aiter_mla._DCPDecodeRoute.CPRR
|
||||
assert md.decode.asm_decode_num_heads, "cprr asm route was not taken"
|
||||
|
||||
o = o.float().view(nr, qlen, gathered_heads, KV_LORA_RANK)
|
||||
shard_lse.append(lse.float().view(nr, qlen, gathered_heads))
|
||||
shard_o.append(o)
|
||||
|
||||
po, _ = _torch_reference(q_f32, kv_f32, ctx, qlen, SCALE, pos)
|
||||
rel = (o - po).abs().max().item() / max(po.abs().max().item(), 1e-6)
|
||||
max_shard_rel = max(max_shard_rel, rel)
|
||||
|
||||
merged, _ = _merge_partials(shard_o, shard_lse)
|
||||
merge_rel = (merged - ref_o).abs().max().item() / max(
|
||||
ref_o.abs().max().item(), 1e-6
|
||||
)
|
||||
return max_shard_rel, merge_rel
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# tests
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@requires_cprr
|
||||
@pytest.mark.parametrize(
|
||||
"heads_per_rank",
|
||||
[
|
||||
16, # 128 gathered heads -- a NATIVE cprr kernel, no padding
|
||||
12, # 96 gathered heads -- K3's DSpark target, must pad up to 128
|
||||
],
|
||||
ids=["native-128", "padded-96-to-128"],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_cprr_shards_match_reference_and_merge_to_full_context(heads_per_rank):
|
||||
"""Every DCP shard must be right, and their LSE merge must reproduce
|
||||
full-context attention.
|
||||
|
||||
A shard that ignored global positions (applying a local causal tail
|
||||
instead) still produces plausible-looking output; it only diverges from the
|
||||
reference on the tokens whose causal window is truncated. Checking the merge
|
||||
alone would hide that, so both are asserted.
|
||||
"""
|
||||
shard_rel, merge_rel = _numerics(heads_per_rank)
|
||||
assert shard_rel < SHARD_RTOL, f"per-shard rel err {shard_rel:.3e}"
|
||||
assert merge_rel < MERGE_RTOL, f"merged rel err {merge_rel:.3e}"
|
||||
|
||||
|
||||
@requires_cprr
|
||||
@torch.inference_mode()
|
||||
def test_wrong_cp_rank_breaks_the_result():
|
||||
"""Positive control: without it, a passing run above proves nothing.
|
||||
|
||||
Feeding shard r's pages to the kernel while telling it that it is rank r+1
|
||||
shifts every reconstructed global position by one, so both the causal mask
|
||||
and the round-robin reconstruction are wrong. If the error does NOT blow up,
|
||||
the kernel is not using cp_rank and the test above has no power.
|
||||
"""
|
||||
shard_rel, merge_rel = _numerics(16, sabotage=True)
|
||||
assert shard_rel > SHARD_RTOL or merge_rel > MERGE_RTOL, (
|
||||
f"sabotaged run still passed (shard {shard_rel:.3e}, merge "
|
||||
f"{merge_rel:.3e}) -- cp_rank is not reaching the kernel"
|
||||
)
|
||||
|
||||
|
||||
@requires_cprr
|
||||
def test_cprr_kernel_is_selected_for_the_k3_shape():
|
||||
"""The K3 DSpark target shape must actually reach the cprr route.
|
||||
|
||||
Guards against a silent fall-out to segmented/Triton, which is what the
|
||||
stock configuration does and what this route exists to avoid.
|
||||
"""
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla as m
|
||||
|
||||
assert (
|
||||
m._asm_dcp_verify_configured(
|
||||
dcp_world_size=DCP, cp_interleave=1, multi_token_decode=True
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert QLEN >= m._MIN_CPRR_QLEN
|
||||
assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR
|
||||
assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS
|
||||
@@ -87,21 +87,31 @@ def _builder(
|
||||
kv_cache_dtype: str = "auto",
|
||||
dcp_world_size: int = 1,
|
||||
dcp_rank: int = 0,
|
||||
asm_dcp_verify: bool = False,
|
||||
decode_causal: bool = True,
|
||||
supports_segmented_dcp_verify: bool | None = None,
|
||||
):
|
||||
decode_num_heads = num_heads * dcp_world_size
|
||||
asm_heads = (
|
||||
rocm_aiter_mla._asm_dcp_verify_heads(decode_num_heads) if asm_dcp_verify else 0
|
||||
)
|
||||
if supports_segmented_dcp_verify is None:
|
||||
supports_segmented_dcp_verify = rocm_aiter_mla._segmented_dcp_verify_supported(
|
||||
dcp_world_size, 1
|
||||
)
|
||||
stub = SimpleNamespace(
|
||||
device=torch.device("cpu"),
|
||||
num_heads=num_heads,
|
||||
# DCP gathers every rank's head shard before decode, and the routing
|
||||
# predicates read the gathered count.
|
||||
_decode_num_heads=num_heads * dcp_world_size,
|
||||
_decode_num_heads=decode_num_heads,
|
||||
dcp_world_size=dcp_world_size,
|
||||
dcp_rank=dcp_rank,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
# Mirrors the production constructor: configuration decides whether the
|
||||
# segmented route is available, query length decides per batch.
|
||||
_supports_segmented_dcp_verify=rocm_aiter_mla._segmented_dcp_verify_supported(
|
||||
dcp_world_size, 1
|
||||
),
|
||||
# segmented route is available, query length decides per batch. CPRR
|
||||
# preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back.
|
||||
_supports_segmented_dcp_verify=supports_segmented_dcp_verify,
|
||||
# Derived once in the real constructor, so derive it once here too.
|
||||
_segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size),
|
||||
_dcp_verify_buffers=None,
|
||||
@@ -122,7 +132,11 @@ def _builder(
|
||||
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
|
||||
_use_persistent_metadata=False,
|
||||
kernel_block_size=kernel_block_size,
|
||||
_num_attention_heads=AiterMLAHelper.get_actual_mla_num_heads(num_heads),
|
||||
_num_attention_heads=(
|
||||
asm_heads
|
||||
if asm_dcp_verify
|
||||
else AiterMLAHelper.get_actual_mla_num_heads(num_heads)
|
||||
),
|
||||
_mla_work_meta_data=torch.empty(1, dtype=torch.int32),
|
||||
_mla_work_info_set=torch.empty(1, dtype=torch.int32),
|
||||
_mla_work_indptr=torch.empty(1, dtype=torch.int32),
|
||||
@@ -132,7 +146,19 @@ def _builder(
|
||||
_mla_q_dtype=torch.bfloat16,
|
||||
_mla_kv_dtype=torch.bfloat16,
|
||||
decode_attn_out_dtype=torch.bfloat16,
|
||||
_decode_causal=True,
|
||||
_decode_causal=decode_causal,
|
||||
# Default off so existing cases keep exercising segmented/persistent.
|
||||
# Pass asm_dcp_verify=True for the CPRR route and its qlen-floor fallback.
|
||||
_asm_dcp_verify=asm_dcp_verify,
|
||||
_asm_dcp_verify_heads=asm_heads,
|
||||
_g_kv_indptr_buf=(
|
||||
torch.zeros(max_decode_rows + 1, dtype=torch.int32)
|
||||
if asm_dcp_verify and dcp_world_size > 1
|
||||
else None
|
||||
),
|
||||
# Only read when the asm route is on; the real builder derives it from
|
||||
# the device CU count.
|
||||
_mla_max_split_per_batch=256,
|
||||
)
|
||||
# Bound to the stub rather than faked: the verify flatten's per-row view is
|
||||
# part of what _build_decode is being tested for.
|
||||
@@ -287,6 +313,209 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch):
|
||||
assert metadata.dcp_verify is not None
|
||||
|
||||
|
||||
def _cprr_decode_batch(builder, *, qlen: int, num_reqs: int = 1):
|
||||
seq_lens = torch.full((num_reqs,), 16, dtype=torch.int32)
|
||||
query_start_loc = torch.arange(0, (num_reqs + 1) * qlen, qlen, dtype=torch.int32)
|
||||
return AiterMLAMetadataBuilder._build_decode(
|
||||
builder,
|
||||
block_table_tensor=torch.zeros(num_reqs, 8, dtype=torch.int32),
|
||||
seq_lens_device=seq_lens,
|
||||
max_seq_len=16,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
query_start_loc_device=query_start_loc,
|
||||
num_decode_tokens=num_reqs * qlen,
|
||||
max_query_len=qlen,
|
||||
dcp_tot_seq_lens_device=seq_lens * 8,
|
||||
)
|
||||
|
||||
|
||||
def test_asm_preferred_qlen2_falls_back_to_segmented(monkeypatch):
|
||||
"""K=4 can clamp to qlen 2; CPRR has no kernel, so use segmented MLA."""
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
|
||||
)
|
||||
|
||||
metadata = _cprr_decode_batch(
|
||||
_builder(
|
||||
mtp_decode_qlen=5,
|
||||
dcp_world_size=8,
|
||||
num_heads=12,
|
||||
asm_dcp_verify=True,
|
||||
kernel_block_size=2,
|
||||
),
|
||||
qlen=2,
|
||||
)
|
||||
|
||||
assert metadata.dcp_verify is not None
|
||||
assert metadata.g_kv_indptr is None
|
||||
assert not metadata.has_persistent_metadata
|
||||
get_mla_metadata_v1.assert_not_called()
|
||||
|
||||
|
||||
def test_asm_preferred_qlen_at_cprr_floor_stays_on_cprr(monkeypatch):
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
|
||||
)
|
||||
|
||||
metadata = _cprr_decode_batch(
|
||||
_builder(
|
||||
mtp_decode_qlen=5,
|
||||
dcp_world_size=8,
|
||||
num_heads=12,
|
||||
asm_dcp_verify=True,
|
||||
),
|
||||
qlen=rocm_aiter_mla._MIN_CPRR_QLEN,
|
||||
)
|
||||
|
||||
assert metadata.dcp_verify is None
|
||||
assert metadata.g_kv_indptr is not None
|
||||
assert metadata.has_persistent_metadata
|
||||
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is True
|
||||
|
||||
|
||||
def test_cprr_capable_single_token_decode_keeps_persistent_metadata(monkeypatch):
|
||||
"""The CPRR qlen floor must not suppress the ordinary qlen-1 schedule."""
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
|
||||
)
|
||||
|
||||
metadata = _cprr_decode_batch(
|
||||
_builder(
|
||||
mtp_decode_qlen=5,
|
||||
dcp_world_size=8,
|
||||
num_heads=12,
|
||||
asm_dcp_verify=True,
|
||||
),
|
||||
qlen=1,
|
||||
)
|
||||
|
||||
assert metadata.g_kv_indptr is None
|
||||
assert metadata.has_persistent_metadata
|
||||
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None
|
||||
|
||||
|
||||
def test_non_causal_dcp_block_uses_plain_mask0_metadata(monkeypatch):
|
||||
"""A non-causal block needs no CPRR global-position window or head pad."""
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
|
||||
)
|
||||
builder = _builder(
|
||||
mtp_decode_qlen=5,
|
||||
dcp_world_size=8,
|
||||
num_heads=12,
|
||||
asm_dcp_verify=True,
|
||||
)
|
||||
builder._decode_causal = False
|
||||
|
||||
metadata = _cprr_decode_batch(builder, qlen=4)
|
||||
|
||||
assert metadata.dcp_verify is None
|
||||
assert metadata.g_kv_indptr is None
|
||||
assert metadata.has_persistent_metadata
|
||||
assert get_mla_metadata_v1.call_args.args[3] == 96
|
||||
assert get_mla_metadata_v1.call_args.args[5] is False
|
||||
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None
|
||||
|
||||
|
||||
def test_asm_qlen2_without_segmented_fails_during_build(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=mock.MagicMock()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
|
||||
_cprr_decode_batch(
|
||||
_builder(
|
||||
mtp_decode_qlen=5,
|
||||
dcp_world_size=8,
|
||||
num_heads=12,
|
||||
asm_dcp_verify=True,
|
||||
supports_segmented_dcp_verify=False,
|
||||
),
|
||||
qlen=2,
|
||||
)
|
||||
|
||||
|
||||
def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
|
||||
"""Process-level CPRR capability must not pad or route a non-causal batch."""
|
||||
captured = {}
|
||||
|
||||
def fake_aiter_decode(q, kv_buffer, out, *args, **kwargs):
|
||||
captured["q_heads"] = q.shape[1]
|
||||
captured["kwargs"] = kwargs
|
||||
return None, torch.zeros(q.shape[0], q.shape[1])
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_get_aiter_mla_decode", lambda: fake_aiter_decode
|
||||
)
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.num_heads = 12
|
||||
impl.dcp_world_size = 8
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.kv_lora_rank = 512
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.scale = 576**-0.5
|
||||
decode = SimpleNamespace(
|
||||
max_qo_len=4,
|
||||
qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
|
||||
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
|
||||
paged_kv_indices=torch.zeros(1, dtype=torch.int32),
|
||||
paged_kv_last_page_len=torch.ones(1, dtype=torch.int32),
|
||||
use_gluon_decode=False,
|
||||
use_gluon_verify=False,
|
||||
dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN,
|
||||
dcp_verify=None,
|
||||
g_kv_indptr=None,
|
||||
has_persistent_metadata=False,
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
asm_decode_num_heads=0,
|
||||
mla_num_kv_splits=0,
|
||||
cp_world_size=8,
|
||||
cp_rank=0,
|
||||
min_kv_seq_len=1,
|
||||
)
|
||||
attn_metadata = SimpleNamespace(decode=decode, causal=False, work_meta_data=None)
|
||||
layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0))
|
||||
q = torch.zeros(4, 96, 576, dtype=torch.bfloat16)
|
||||
|
||||
output, lse = impl.forward_mqa(q, torch.zeros(1, 1, 576), attn_metadata, layer)
|
||||
|
||||
assert captured["q_heads"] == 96
|
||||
assert captured["kwargs"]["causal"] is False
|
||||
assert "g_kv_indptr" not in captured["kwargs"]
|
||||
assert "num_kv_splits" not in captured["kwargs"]
|
||||
assert output.shape[1] == 96
|
||||
assert lse is not None and lse.shape == (4, 96)
|
||||
|
||||
|
||||
def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
|
||||
"""Single-token DCP decode must come back with an LSE the merge can use.
|
||||
|
||||
@@ -325,9 +554,14 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
|
||||
paged_kv_last_page_len=torch.ones(num_tokens, dtype=torch.int32),
|
||||
use_gluon_decode=False,
|
||||
use_gluon_verify=False,
|
||||
dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN,
|
||||
dcp_verify=None,
|
||||
has_persistent_metadata=False,
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
# 0 == the round-robin (cprr) asm route is off, this rank's decode takes
|
||||
# the existing path. Mirrors the AiterMLADecodeMetadata default.
|
||||
asm_decode_num_heads=0,
|
||||
g_kv_indptr=None,
|
||||
)
|
||||
attn_metadata = SimpleNamespace(decode=decode, causal=True, work_meta_data=None)
|
||||
layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0))
|
||||
@@ -453,6 +687,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
|
||||
),
|
||||
use_gluon_decode=False,
|
||||
use_gluon_verify=False,
|
||||
dcp_route=rocm_aiter_mla._DCPDecodeRoute.SEGMENTED,
|
||||
dcp_verify=view,
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
@@ -145,6 +145,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
|
||||
VLLM_ROCM_USE_AITER_MLA: bool = True
|
||||
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
|
||||
VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["asm", "segmented"] = "segmented"
|
||||
VLLM_ROCM_USE_AITER_MHA: bool = True
|
||||
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False
|
||||
VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False
|
||||
@@ -1309,6 +1310,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ROCM_USE_AITER_MLA": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MLA", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Kernel for causal multi-token (spec-decode) verify steps under decode
|
||||
# context parallelism on gfx950: "asm" uses AITER's round-robin ASM
|
||||
# decode, "segmented" the Triton segmented MLA path.
|
||||
"VLLM_ROCM_AITER_MLA_DCP_VERIFY": env_with_choices(
|
||||
"VLLM_ROCM_AITER_MLA_DCP_VERIFY",
|
||||
"segmented",
|
||||
["asm", "segmented"],
|
||||
case_sensitive=False,
|
||||
),
|
||||
# Small-head (<16) AITER MLA decode kernel selection. Small head counts
|
||||
# (e.g. Kimi-K3: 12 heads/rank at TP8, 6 at TP16) can decode either through
|
||||
# the Gluon small-head kernel or through the padded persistent-scheduling
|
||||
|
||||
@@ -3,11 +3,13 @@
|
||||
|
||||
import functools
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, ClassVar, Final
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
@@ -239,6 +241,78 @@ def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) ->
|
||||
)
|
||||
|
||||
|
||||
# DCP decode routing
|
||||
#
|
||||
# With decode context parallelism each rank holds a round-robin shard of the KV
|
||||
# cache (token t on rank t % dcp_world_size; requires interleave size 1) and
|
||||
# attends over it with all DCP-gathered query heads. Partial outputs are merged
|
||||
# across ranks by LSE. Each decode batch takes one route:
|
||||
#
|
||||
# * plain: qlen == 1 or a non-causal block. Every query sees every local token.
|
||||
# * cprr: causal qlen >= _MIN_CPRR_QLEN, gfx950, speculative decoding. AITER's
|
||||
# round-robin ASM decode rebuilds global positions from g_kv_indptr, cp_rank
|
||||
# and cp_world_size and applies the causal window in-kernel. Kernels exist only
|
||||
# at _NATIVE_CPRR_HEADS; other gathered head counts are padded up.
|
||||
# * segmented: causal qlen > 1 when cprr is not selected or qlen is below
|
||||
# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV length
|
||||
# encodes its causal window.
|
||||
#
|
||||
# The plain kernel must never serve a causal qlen > 1 block: it applies
|
||||
# causality on local indices, which is wrong over a round-robin shard.
|
||||
_NATIVE_CPRR_HEADS: Final = (16, 32, 64, 128)
|
||||
_MIN_CPRR_QLEN: Final = 3
|
||||
|
||||
|
||||
class _DCPDecodeRoute(Enum):
|
||||
PLAIN = "plain"
|
||||
SEGMENTED = "segmented"
|
||||
CPRR = "cprr"
|
||||
|
||||
|
||||
def _asm_dcp_verify_heads(decode_num_heads: int) -> int:
|
||||
"""Head count the cprr asm decode runs at, or 0 if it cannot serve this shape."""
|
||||
if decode_num_heads in _NATIVE_CPRR_HEADS:
|
||||
return decode_num_heads
|
||||
for nat in _NATIVE_CPRR_HEADS:
|
||||
if nat >= decode_num_heads:
|
||||
return nat
|
||||
return 0
|
||||
|
||||
|
||||
def _select_dcp_decode_route(
|
||||
supports_segmented: bool,
|
||||
causal: bool,
|
||||
max_qo_len: int,
|
||||
asm_selected: bool,
|
||||
) -> _DCPDecodeRoute:
|
||||
"""Select one DCP decode route for this batch; see DCP decode routing."""
|
||||
if not causal or max_qo_len <= 1:
|
||||
return _DCPDecodeRoute.PLAIN
|
||||
if asm_selected and max_qo_len >= _MIN_CPRR_QLEN:
|
||||
return _DCPDecodeRoute.CPRR
|
||||
if supports_segmented:
|
||||
return _DCPDecodeRoute.SEGMENTED
|
||||
raise RuntimeError(
|
||||
"ROCM_AITER_MLA DCP multi-token verify requires either segmented "
|
||||
"MLA or the round-robin asm decode."
|
||||
)
|
||||
|
||||
|
||||
def _asm_dcp_verify_configured(
|
||||
dcp_world_size: int, cp_interleave: int, multi_token_decode: bool
|
||||
) -> bool:
|
||||
"""Whether asm cprr DCP verify is reachable in this process.
|
||||
|
||||
Requires DCP with interleave size 1, speculative decoding, and gfx950 (the
|
||||
only arch AITER builds cprr kernels for).
|
||||
"""
|
||||
if not (dcp_world_size > 1 and cp_interleave == 1 and multi_token_decode):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return bool(on_gfx950())
|
||||
|
||||
|
||||
def _aiter_mla_small_head_mode() -> str:
|
||||
"""Small-head (<16) MLA decode kernel selection.
|
||||
|
||||
@@ -410,8 +484,19 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
|
||||
max_qo_len: int | None = None
|
||||
# Minimum KV length used by Gluon to choose a safe split count.
|
||||
min_kv_seq_len: int = 1
|
||||
# Exactly one DCP decode route is selected by the metadata builder.
|
||||
dcp_route: _DCPDecodeRoute = _DCPDecodeRoute.PLAIN
|
||||
# Set exactly when this batch routes to segmented DCP verification.
|
||||
dcp_verify: AiterMLADCPVerifyMetadata | None = None
|
||||
# cprr only: global per-request page indptr and this rank's position.
|
||||
g_kv_indptr: torch.Tensor | None = None
|
||||
cp_world_size: int = 1
|
||||
cp_rank: int = 0
|
||||
# Padded head count cprr runs at; 0 when this batch is not on cprr.
|
||||
asm_decode_num_heads: int = 0
|
||||
# Must equal the split cap the reduce scratch was sized with, or the kernel
|
||||
# writes past it and faults.
|
||||
mla_num_kv_splits: int = 0
|
||||
# Small-head decode uses Gluon (avoids padding to 16).
|
||||
use_gluon_decode: bool = False
|
||||
# Small-head non-DCP multi-token verify uses Gluon's native MTP entry.
|
||||
@@ -460,6 +545,9 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
# Set from the common metadata every build; a batch is causal unless the
|
||||
# drafter says otherwise.
|
||||
_decode_causal: bool = True
|
||||
# A non-causal draft needs no cross-shard causal window: the ordinary
|
||||
# mask0 decode returns per-row LSE for the DCP merge.
|
||||
supports_non_causal_multi_token_dcp: ClassVar[bool] = True
|
||||
|
||||
@staticmethod
|
||||
def _uniform_padded_mtp_qo_len(
|
||||
@@ -513,15 +601,48 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
parallel_config.decode_context_parallel_size,
|
||||
parallel_config.cp_kv_cache_interleave_size,
|
||||
)
|
||||
asm_dcp_verify_config = (
|
||||
_asm_dcp_verify_configured(
|
||||
parallel_config.decode_context_parallel_size,
|
||||
parallel_config.cp_kv_cache_interleave_size,
|
||||
multi_token_decode=vllm_config.speculative_config is not None,
|
||||
)
|
||||
and envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "asm"
|
||||
)
|
||||
super().__init__(
|
||||
kv_cache_spec,
|
||||
layer_names,
|
||||
vllm_config,
|
||||
device,
|
||||
AiterMLAMetadata,
|
||||
supports_dcp_with_varlen=supports_segmented_dcp_verify,
|
||||
supports_dcp_with_varlen=(
|
||||
supports_segmented_dcp_verify or asm_dcp_verify_config
|
||||
),
|
||||
)
|
||||
self._asm_dcp_verify = False
|
||||
self._asm_dcp_verify_heads = 0
|
||||
# The head-count half of the route decision needs self.num_heads, so only
|
||||
# the config half is known before super().__init__().
|
||||
if asm_dcp_verify_config:
|
||||
self._asm_dcp_verify_heads = _asm_dcp_verify_heads(
|
||||
self.num_heads * self.dcp_world_size
|
||||
)
|
||||
if not self._asm_dcp_verify_heads:
|
||||
raise ValueError(
|
||||
"VLLM_ROCM_AITER_MLA_DCP_VERIFY=asm, but the round-robin asm "
|
||||
f"decode has no kernel for {self.num_heads * self.dcp_world_size} "
|
||||
f"DCP-gathered heads (native counts: {_NATIVE_CPRR_HEADS}). "
|
||||
"Set VLLM_ROCM_AITER_MLA_DCP_VERIFY=segmented."
|
||||
)
|
||||
self._asm_dcp_verify = True
|
||||
self._supports_segmented_dcp_verify = supports_segmented_dcp_verify
|
||||
self._mla_max_split_per_batch = 0
|
||||
if self._asm_dcp_verify:
|
||||
# Cap on KV splits per batch; at low batch the KV axis is the only
|
||||
# parallelism. Buffer sizing and every runtime call must see this value.
|
||||
self._mla_max_split_per_batch = torch.cuda.get_device_properties(
|
||||
device
|
||||
).multi_processor_count
|
||||
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
self.decode_attn_out_dtype = vllm_config.model_config.dtype
|
||||
@@ -576,6 +697,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads(
|
||||
self._decode_num_heads
|
||||
)
|
||||
# the cprr kernel exists only at the native head
|
||||
# counts, so run (and size the persistent schedule) at the padded count.
|
||||
# 96 is 16-aligned, so get_actual_mla_num_heads leaves it alone and the
|
||||
# pad has to be applied here explicitly.
|
||||
if self._asm_dcp_verify:
|
||||
self._num_attention_heads = self._asm_dcp_verify_heads
|
||||
kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")
|
||||
if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"):
|
||||
kv_cache_dtype_str = "fp8"
|
||||
@@ -619,6 +746,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
kv_dtype,
|
||||
is_sparse=False,
|
||||
fast_mode=True,
|
||||
# Must match max_split_per_batch in get_mla_metadata_v1; without it
|
||||
# aiter sizes the reduce scratch from capacity (~9 GiB).
|
||||
**(
|
||||
dict(max_split_per_batch=self._mla_max_split_per_batch)
|
||||
if self._asm_dcp_verify
|
||||
else {}
|
||||
),
|
||||
)
|
||||
self._mla_work_meta_data = torch.empty(
|
||||
work_meta_data_size, dtype=work_meta_data_type, device=device
|
||||
@@ -664,6 +798,14 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
device,
|
||||
)
|
||||
|
||||
# Full cudagraphs need a stable address for the cprr global page indptr.
|
||||
# Zero-initialized; element 0 is the indptr base and is never rewritten.
|
||||
self._g_kv_indptr_buf: torch.Tensor | None = None
|
||||
if self._asm_dcp_verify:
|
||||
self._g_kv_indptr_buf = torch.zeros(
|
||||
max_num_reqs + 1, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
# Persistent buffers for segmented DCP verification. Captured graphs
|
||||
# require stable addresses while row lengths vary between replays.
|
||||
self._dcp_verify_buffers: AiterMLADCPVerifyMetadata | None = None
|
||||
@@ -683,8 +825,11 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
)
|
||||
|
||||
if self._supports_segmented_dcp_verify and self._mtp_decode_qlen > 1:
|
||||
# A DCP rank's shard of the longest sequence bounds every verify
|
||||
# row, and full graphs need that bound to be constant.
|
||||
# Allocate even when CPRR is the preferred route: a later
|
||||
# replay can fall below _MIN_CPRR_QLEN and needs these
|
||||
# addresses. A DCP rank's shard of the longest sequence
|
||||
# bounds every verify row, and full graphs need that bound
|
||||
# to be constant.
|
||||
num_dcp_partitions = (
|
||||
self.dcp_world_size * self.cp_kv_cache_interleave_size
|
||||
)
|
||||
@@ -1063,6 +1208,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
pad_uniform_mtp = padded_mtp_qo_len > 0
|
||||
|
||||
seq_lens_for_kernel = seq_lens_device
|
||||
# the global lengths need the same dummy-row
|
||||
# treatment as the local ones below, but only for g_kv_indptr -- the
|
||||
# metadata field keeps the true values, which is what the segmented
|
||||
# route wants (it derives its own row lens and gives a dummy row
|
||||
# length 0, correctly).
|
||||
g_tot_seq_lens = dcp_tot_seq_lens_device
|
||||
num_kernel_reqs = num_reqs
|
||||
if pad_uniform_mtp:
|
||||
qo_lens_device = (
|
||||
@@ -1074,6 +1225,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
seq_lens_for_kernel,
|
||||
seq_lens_for_kernel.new_full((), max_qo_len),
|
||||
)
|
||||
# A dummy row just got a LOCAL length of max_qo_len, so its GLOBAL
|
||||
# length has to say the same thing or the cprr mask is degenerate:
|
||||
# global_len 0 puts the row query positions at {-3,-2,-1} while its
|
||||
# local shard sits at {rank, W+rank, 2W+rank}, so every entry is
|
||||
# masked and the row softmaxes over nothing. max_qo_len * W is
|
||||
# exactly the global length whose round-robin shard is max_qo_len
|
||||
# on every rank.
|
||||
if g_tot_seq_lens is not None:
|
||||
g_tot_seq_lens = torch.where(
|
||||
qo_lens_device > 0,
|
||||
g_tot_seq_lens,
|
||||
g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size),
|
||||
)
|
||||
|
||||
if self._graph_seq_lens is not None:
|
||||
self._graph_seq_lens[:num_kernel_reqs].copy_(
|
||||
@@ -1106,16 +1270,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
causal,
|
||||
self._kv_cache_bytes,
|
||||
)
|
||||
use_segmented_dcp_verify = (
|
||||
self._supports_segmented_dcp_verify and max_qo_len > 1
|
||||
)
|
||||
dcp_route = _DCPDecodeRoute.PLAIN
|
||||
if self.dcp_world_size > 1:
|
||||
dcp_route = _select_dcp_decode_route(
|
||||
self._supports_segmented_dcp_verify,
|
||||
causal,
|
||||
int(max_qo_len),
|
||||
self._asm_dcp_verify,
|
||||
)
|
||||
|
||||
# Segmented DCP verify carries its own per-row subpage table, so the
|
||||
# flat per-token view is dead work for it. Leave the buffer alone and
|
||||
# hand the metadata None, so a future reader cannot pick up whatever
|
||||
# the previous batch left behind.
|
||||
paged_kv_indices = None
|
||||
if not use_segmented_dcp_verify:
|
||||
if dcp_route is not _DCPDecodeRoute.SEGMENTED:
|
||||
if self.compilation_config.cudagraph_mode.has_full_cudagraphs():
|
||||
self.paged_kv_indices.fill_(-1)
|
||||
|
||||
@@ -1169,6 +1338,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
else:
|
||||
qo_indptr = query_start_loc_device[: 1 + num_kernel_reqs]
|
||||
|
||||
g_kv_indptr = None
|
||||
if dcp_route is _DCPDecodeRoute.CPRR and g_tot_seq_lens is not None:
|
||||
assert self._g_kv_indptr_buf is not None
|
||||
_ngk = g_tot_seq_lens.shape[0]
|
||||
g_kv_indptr = self._g_kv_indptr_buf[: _ngk + 1]
|
||||
g_kv_indptr[1:].copy_(g_tot_seq_lens.cumsum(dim=0, dtype=torch.int32))
|
||||
# A FULL cudagraph replays at its capture-time request count, so a
|
||||
# smaller real batch leaves the tail holding the PREVIOUS step's
|
||||
# cumsum. Flatten it the way paged_kv_indptr is flattened below, so
|
||||
# padding rows read global_len == 0 and agree with their zero-length
|
||||
# local shard instead of a stale global length.
|
||||
self._g_kv_indptr_buf[_ngk + 1 :].fill_(g_kv_indptr[-1])
|
||||
|
||||
has_persistent_metadata = False
|
||||
# Only the asm decode consumes the schedule, so gate on the routing
|
||||
# rather than on num_heads >= 16, which denies it to a padded rank
|
||||
@@ -1178,7 +1360,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
use_persistent_metadata = (
|
||||
not use_gluon_decode
|
||||
and not use_gluon_verify
|
||||
and not use_segmented_dcp_verify
|
||||
and dcp_route is not _DCPDecodeRoute.SEGMENTED
|
||||
# A padded rank has no bf16 persistent kernel past qlen 4 where the
|
||||
# gfx950 fold is absent; the non-persistent entry covers it. fp8
|
||||
# keeps the schedule -- its fold rejects non-persistent outright.
|
||||
@@ -1218,13 +1400,28 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
uni_qo_len = (
|
||||
max_qo_len if pad_uniform_mtp or torch.all(qo_len == max_qo_len) else -1
|
||||
)
|
||||
cprr_kwargs: dict = {}
|
||||
is_causal = causal
|
||||
if self._asm_dcp_verify:
|
||||
cprr_kwargs = dict(
|
||||
max_split_per_batch=self._mla_max_split_per_batch,
|
||||
intra_batch_mode=False,
|
||||
)
|
||||
if dcp_route is _DCPDecodeRoute.CPRR:
|
||||
is_causal = False
|
||||
cprr_kwargs["is_cp_round_robin"] = True
|
||||
metadata_num_heads = (
|
||||
self._asm_dcp_verify_heads
|
||||
if dcp_route is _DCPDecodeRoute.CPRR
|
||||
else AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads)
|
||||
)
|
||||
get_mla_metadata_v1(
|
||||
qo_indptr,
|
||||
paged_kv_indptr,
|
||||
paged_kv_last_page_len,
|
||||
self._num_attention_heads,
|
||||
metadata_num_heads,
|
||||
1,
|
||||
causal,
|
||||
is_causal,
|
||||
self._mla_work_meta_data,
|
||||
self._mla_work_info_set,
|
||||
self._mla_work_indptr,
|
||||
@@ -1238,6 +1435,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
fast_mode=True,
|
||||
dtype_q=self._mla_q_dtype,
|
||||
dtype_kv=self._mla_kv_dtype,
|
||||
**cprr_kwargs,
|
||||
)
|
||||
has_persistent_metadata = True
|
||||
|
||||
@@ -1261,7 +1459,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
min_kv_seq_len = int(per_req_len.min().item())
|
||||
|
||||
dcp_verify = None
|
||||
if use_segmented_dcp_verify:
|
||||
if dcp_route is _DCPDecodeRoute.SEGMENTED:
|
||||
assert dcp_tot_seq_lens_device is not None
|
||||
dcp_verify = self._build_dcp_verify_row_view(
|
||||
int(max_qo_len),
|
||||
@@ -1279,7 +1477,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
|
||||
max_qo_len=max_qo_len,
|
||||
min_kv_seq_len=min_kv_seq_len,
|
||||
dcp_route=dcp_route,
|
||||
dcp_verify=dcp_verify,
|
||||
g_kv_indptr=g_kv_indptr,
|
||||
cp_world_size=self.dcp_world_size,
|
||||
cp_rank=self.dcp_rank,
|
||||
asm_decode_num_heads=(
|
||||
self._asm_dcp_verify_heads if dcp_route is _DCPDecodeRoute.CPRR else 0
|
||||
),
|
||||
mla_num_kv_splits=(
|
||||
self._mla_max_split_per_batch
|
||||
if dcp_route is _DCPDecodeRoute.CPRR
|
||||
else 0
|
||||
),
|
||||
use_gluon_decode=use_gluon_decode,
|
||||
use_gluon_verify=use_gluon_verify,
|
||||
attn_out_dtype=self.decode_attn_out_dtype,
|
||||
@@ -2062,8 +2272,10 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
)
|
||||
return o, None
|
||||
|
||||
verify = decode.dcp_verify
|
||||
if verify is not None:
|
||||
dcp_route = decode.dcp_route
|
||||
if dcp_route is _DCPDecodeRoute.SEGMENTED:
|
||||
verify = decode.dcp_verify
|
||||
assert verify is not None
|
||||
if type(q) is tuple:
|
||||
q_nope, q_pe = q
|
||||
else:
|
||||
@@ -2085,11 +2297,6 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
decode.attn_out_dtype,
|
||||
)
|
||||
|
||||
if self.dcp_world_size > 1 and int(decode.max_qo_len) > 1:
|
||||
raise RuntimeError(
|
||||
"ROCM_AITER_MLA DCP multi-token verify requires segmented MLA."
|
||||
)
|
||||
|
||||
if type(q) is tuple:
|
||||
q = torch.cat(q, dim=-1)
|
||||
|
||||
@@ -2102,8 +2309,19 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
"ROCM_AITER_MLA decode expected the DCP-gathered query head count "
|
||||
f"{self._decode_num_heads}, got {q.shape[1]}"
|
||||
)
|
||||
mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q)
|
||||
mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads)
|
||||
# cprr has kernels only at native head counts, so pad q up (e.g. 96 ->
|
||||
# 128). Heads are independent in MLA; the extras are sliced off below.
|
||||
asm_dcp_heads = decode.asm_decode_num_heads
|
||||
if asm_dcp_heads > self._decode_num_heads:
|
||||
mla_num_heads = asm_dcp_heads
|
||||
mla_padded_q = AiterMLAHelper.get_mla_padded_q(
|
||||
self._decode_num_heads, q, target_heads=mla_num_heads
|
||||
)
|
||||
else:
|
||||
mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q)
|
||||
mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads(
|
||||
self._decode_num_heads
|
||||
)
|
||||
o = torch.empty(
|
||||
B,
|
||||
mla_num_heads,
|
||||
@@ -2137,6 +2355,16 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
|
||||
lse = None
|
||||
if self.dcp_world_size > 1:
|
||||
if dcp_route is _DCPDecodeRoute.CPRR:
|
||||
# cprr only: global positions for the in-kernel causal window.
|
||||
assert decode.g_kv_indptr is not None
|
||||
mla_kwargs["g_kv_indptr"] = decode.g_kv_indptr
|
||||
mla_kwargs["cp_world_size"] = decode.cp_world_size
|
||||
mla_kwargs["cp_rank"] = decode.cp_rank
|
||||
# Must match the schedule get_mla_metadata_v1 was built with.
|
||||
assert decode.mla_num_kv_splits > 0
|
||||
mla_kwargs["num_kv_splits"] = decode.mla_num_kv_splits
|
||||
mla_kwargs["intra_batch_mode"] = False
|
||||
# The vLLM custom-op wrapper exposes only the in-place output and
|
||||
# drops aiter's final LSE, which the cross-shard merge needs, so go
|
||||
# through aiter's native entry point on the DCP path.
|
||||
@@ -2151,6 +2379,7 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
decode.max_qo_len,
|
||||
sm_scale=self.scale,
|
||||
return_lse=True,
|
||||
causal=attn_metadata.causal,
|
||||
**mla_kwargs,
|
||||
)
|
||||
assert lse is not None, (
|
||||
@@ -2172,7 +2401,13 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
**mla_kwargs,
|
||||
)
|
||||
|
||||
output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o)
|
||||
if lse is not None:
|
||||
lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse)
|
||||
if asm_dcp_heads > self._decode_num_heads:
|
||||
# Keep a view: a post-kernel copy races the symm-mem a2a combine.
|
||||
output = o[:, : self._decode_num_heads]
|
||||
if lse is not None:
|
||||
lse = lse[:, : self._decode_num_heads, ...]
|
||||
else:
|
||||
output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o)
|
||||
if lse is not None:
|
||||
lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse)
|
||||
return output, lse
|
||||
|
||||
Reference in New Issue
Block a user