[ROCm][MLA] Route gfx942 DCP causal multi-token verify to Triton split-KV decode (#58555)
Signed-off-by: Xavier Aguilar <xavier.aguilarfruto@amd.com>
This commit is contained in:
@@ -161,6 +161,11 @@ def _build(monkeypatch, route, gathered_heads, segmented_supported=True):
|
||||
lambda *args: segmented_supported,
|
||||
)
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_fp8_mla_prefill_supported", lambda: False)
|
||||
# These cases pick between cprr and segmented. On a gfx942 runner the real
|
||||
# arch would add the Triton verify route, which this stub cannot build.
|
||||
import vllm.platforms.rocm as rocm
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx942", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "get_dcp_group", lambda: SimpleNamespace(rank_in_group=0)
|
||||
)
|
||||
@@ -264,25 +269,46 @@ def test_min_cprr_qlen_is_above_two():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"supports,causal,qlen,asm,expected",
|
||||
"segmented,triton,causal,qlen,asm,expected",
|
||||
[
|
||||
(True, True, 1, True, Route.PLAIN),
|
||||
(True, True, 2, True, Route.SEGMENTED),
|
||||
(True, True, 3, True, Route.CPRR),
|
||||
(True, True, 4, True, Route.CPRR),
|
||||
(True, True, 4, False, Route.SEGMENTED),
|
||||
(True, False, 2, True, Route.PLAIN),
|
||||
(True, False, 4, False, Route.PLAIN),
|
||||
(False, True, 3, True, Route.CPRR),
|
||||
(True, False, True, 1, True, Route.PLAIN),
|
||||
(True, False, True, 2, True, Route.SEGMENTED),
|
||||
(True, False, True, 3, True, Route.CPRR),
|
||||
(True, False, True, 4, True, Route.CPRR),
|
||||
(True, False, True, 4, False, Route.SEGMENTED),
|
||||
(True, False, False, 2, True, Route.PLAIN),
|
||||
(True, False, False, 4, False, Route.PLAIN),
|
||||
(False, False, True, 3, True, Route.CPRR),
|
||||
# gfx942: no cprr and no segmented, so causal verify goes to Triton.
|
||||
(False, True, True, 1, False, Route.PLAIN),
|
||||
(False, True, True, 2, False, Route.TRITON),
|
||||
(False, True, True, 4, False, Route.TRITON),
|
||||
(False, True, False, 4, False, Route.PLAIN),
|
||||
# cprr still wins at or above its qlen floor.
|
||||
(False, True, True, 4, True, Route.CPRR),
|
||||
(False, True, True, 2, True, Route.TRITON),
|
||||
],
|
||||
)
|
||||
def test_dcp_decode_route(supports, causal, qlen, asm, expected):
|
||||
assert _select_route(supports, causal, qlen, asm) is expected
|
||||
def test_dcp_decode_route(segmented, triton, causal, qlen, asm, expected):
|
||||
route = _select_route(
|
||||
supports_segmented=segmented,
|
||||
supports_triton=triton,
|
||||
causal=causal,
|
||||
max_qo_len=qlen,
|
||||
asm_selected=asm,
|
||||
)
|
||||
assert route is expected
|
||||
|
||||
|
||||
def test_causal_multi_token_batch_without_a_valid_route_fails():
|
||||
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
|
||||
_select_route(False, True, 2, True)
|
||||
_select_route(
|
||||
supports_segmented=False,
|
||||
supports_triton=False,
|
||||
causal=True,
|
||||
max_qo_len=2,
|
||||
asm_selected=True,
|
||||
)
|
||||
|
||||
|
||||
# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs
|
||||
|
||||
@@ -536,5 +536,12 @@ def test_cprr_kernel_is_selected_for_the_k3_shape():
|
||||
is True
|
||||
)
|
||||
assert QLEN >= m._MIN_CPRR_QLEN
|
||||
assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR
|
||||
route = m._select_dcp_decode_route(
|
||||
supports_segmented=True,
|
||||
supports_triton=False,
|
||||
causal=True,
|
||||
max_qo_len=QLEN,
|
||||
asm_selected=True,
|
||||
)
|
||||
assert route is m._DCPDecodeRoute.CPRR
|
||||
assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Causal target verify under DCP on gfx942 (MI300/MI325).
|
||||
|
||||
Reproduces the attention shape of
|
||||
|
||||
vllm serve moonshotai/Kimi-K3 --tensor-parallel-size 8 \
|
||||
--decode-context-parallel-size 8 \
|
||||
--speculative-config '{"method": "dspark", "num_speculative_tokens": 5}'
|
||||
|
||||
96 gathered heads, bf16 KV, qlen 6 and the 768-token blocks the hybrid KDA
|
||||
layers impose. Segmented MLA tiles those blocks at 128 tokens and needs 128 KiB
|
||||
of LDS (gfx942 has 64 KiB), so the builder must route this shape to the generic
|
||||
Triton split-KV kernel. DeepSeek-V3 with MTP and an fp8 KV cache at DCP 8 hits
|
||||
the same limit through its 128 gathered heads. Shapes segmented MLA fits stay
|
||||
on it.
|
||||
|
||||
The real metadata builder and ``forward_mqa`` run on one GPU with the DCP ranks
|
||||
simulated one after another. That is sound because this path has no
|
||||
collective: the query all-gather happens in the layer, before ``forward_mqa``.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm AITER MLA tests", allow_module_level=True)
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found # noqa: E402
|
||||
|
||||
# DeepSeek-R1 supplies the MLA dims, which Kimi-K3 and DeepSeek-V3 share; only
|
||||
# the head count is overridden.
|
||||
MODEL = "deepseek-ai/DeepSeek-R1"
|
||||
KV_LORA_RANK, ROPE_DIM, NOPE_DIM = 512, 64, 128
|
||||
HEAD_SIZE = KV_LORA_RANK + ROPE_DIM
|
||||
SCALE = (NOPE_DIM + ROPE_DIM) ** -0.5
|
||||
DCP = 8
|
||||
# Each rank holds ~800 tokens per request, so its shard spans several blocks.
|
||||
NUM_REQS, CTX = 4, 6400
|
||||
FP8_KV_SCALE = 0.05
|
||||
|
||||
|
||||
def _on_gfx942() -> bool:
|
||||
if not (is_aiter_found() and torch.accelerator.is_available()):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx942
|
||||
|
||||
return on_gfx942()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(not _on_gfx942(), reason="needs ROCm + AITER on gfx942")
|
||||
|
||||
|
||||
class _FakeGroup:
|
||||
def __init__(self, world_size: int, rank: int):
|
||||
self.world_size = world_size
|
||||
self.rank_in_group = self.rank = self.local_rank = rank
|
||||
|
||||
def all_gather(self, tensor, dim=0):
|
||||
raise RuntimeError("DCP verify must not issue a collective")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_dcp_groups(monkeypatch):
|
||||
"""Point every importer of get_dcp_group at a settable rank stub."""
|
||||
import vllm.distributed.parallel_state as ps
|
||||
import vllm.model_executor.layers.attention.mla_attention as mla_attention
|
||||
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
|
||||
|
||||
group = _FakeGroup(DCP, 0)
|
||||
for module in (ps, mla_attention, backend):
|
||||
monkeypatch.setattr(module, "get_dcp_group", lambda: group)
|
||||
monkeypatch.setattr(ps, "get_tp_group", lambda: _FakeGroup(1, 0))
|
||||
return group
|
||||
|
||||
|
||||
def _vllm_config(heads_per_rank: int, qlen: int, fp8_kv: bool):
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
|
||||
config = create_vllm_config(
|
||||
model_name=MODEL,
|
||||
max_model_len=CTX + qlen,
|
||||
max_num_seqs=NUM_REQS,
|
||||
max_num_batched_tokens=4096,
|
||||
)
|
||||
config.model_config.model_arch_config.total_num_attention_heads = heads_per_rank
|
||||
# The reorder threshold reads only these two fields; without them a
|
||||
# multi-token row is classified as prefill and never reaches decode.
|
||||
config.speculative_config = SimpleNamespace(
|
||||
num_speculative_tokens=qlen - 1, parallel_drafting=False
|
||||
)
|
||||
config.parallel_config.decode_context_parallel_size = DCP
|
||||
config.parallel_config.cp_kv_cache_interleave_size = 1
|
||||
if fp8_kv:
|
||||
config.cache_config.cache_dtype = "fp8"
|
||||
return config
|
||||
|
||||
|
||||
def _ctx_layer():
|
||||
"""The registered-layer fields MLACommonMetadataBuilder.__init__ reads."""
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
|
||||
dcp_manager = object.__new__(MLADCPManager)
|
||||
dcp_manager.init_kv_gather = lambda *args, **kwargs: None
|
||||
# Only chunked-context prefill uses the cloned backend.
|
||||
prefill_backend = SimpleNamespace()
|
||||
prefill_backend.clone = lambda: prefill_backend
|
||||
return SimpleNamespace(
|
||||
non_causal_multi_token_decode=False,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=NOPE_DIM,
|
||||
qk_rope_head_dim=ROPE_DIM,
|
||||
v_head_dim=NOPE_DIM,
|
||||
dcp_manager=dcp_manager,
|
||||
prefill_backend=prefill_backend,
|
||||
)
|
||||
|
||||
|
||||
def _impl(dcp_rank: int, heads_per_rank: int, fp8_kv: bool):
|
||||
"""forward_mqa needs these attributes, not the weight-loading __init__."""
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.num_heads = heads_per_rank
|
||||
impl.kv_lora_rank = KV_LORA_RANK
|
||||
impl.qk_rope_head_dim = ROPE_DIM
|
||||
impl.dcp_world_size = DCP
|
||||
impl.dcp_rank = dcp_rank
|
||||
impl.pcp_world_size = 1
|
||||
impl.kv_cache_dtype = "fp8" if fp8_kv else "auto"
|
||||
impl.scale = SCALE
|
||||
impl._sm_count = current_platform.num_compute_units()
|
||||
return impl
|
||||
|
||||
|
||||
def _run_rank(config, spec, layer_name, group, rank, q, kv_cache_src, k_scale):
|
||||
"""Shard the KV round-robin onto ``rank`` and run its verify step."""
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
|
||||
|
||||
device = q.device
|
||||
qlen, heads = q.shape[1], q.shape[2] // DCP
|
||||
seq_len = CTX + qlen
|
||||
positions = torch.arange(rank, seq_len, DCP, device=device)
|
||||
local_len = positions.numel()
|
||||
# Production block tables span max_model_len, so every rank's table is at
|
||||
# least as wide as the largest shard needs.
|
||||
blocks_per_req = cdiv(cdiv(seq_len, DCP), spec.block_size)
|
||||
kv_cache = torch.zeros(
|
||||
NUM_REQS * blocks_per_req,
|
||||
spec.block_size,
|
||||
HEAD_SIZE,
|
||||
dtype=kv_cache_src.dtype,
|
||||
device=device,
|
||||
)
|
||||
kv_cache.view(NUM_REQS, -1, HEAD_SIZE)[:, :local_len] = kv_cache_src[:, positions]
|
||||
block_table = torch.arange(
|
||||
NUM_REQS * blocks_per_req, dtype=torch.int32, device=device
|
||||
).view(NUM_REQS, blocks_per_req)
|
||||
|
||||
query_start_loc = torch.arange(
|
||||
0, NUM_REQS * qlen + 1, qlen, dtype=torch.int32, device=device
|
||||
)
|
||||
local_lens = torch.full((NUM_REQS,), local_len, dtype=torch.int32)
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.full((NUM_REQS,), seq_len, dtype=torch.int32, device=device),
|
||||
num_reqs=NUM_REQS,
|
||||
num_actual_tokens=NUM_REQS * qlen,
|
||||
max_query_len=qlen,
|
||||
max_seq_len=seq_len,
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=torch.arange(NUM_REQS * qlen, dtype=torch.int64, device=device),
|
||||
causal=True,
|
||||
dcp_local_seq_lens=local_lens.to(device),
|
||||
dcp_local_seq_lens_cpu_upper_bound=local_lens,
|
||||
)
|
||||
|
||||
group.rank_in_group = group.rank = group.local_rank = rank
|
||||
builder = AiterMLAMetadataBuilder(spec, [layer_name], config, device)
|
||||
metadata = builder.build(0, common)
|
||||
layer = SimpleNamespace(
|
||||
_q_scale=torch.tensor(1.0, device=device),
|
||||
_k_scale=torch.tensor(k_scale, device=device),
|
||||
)
|
||||
impl = _impl(rank, heads, kv_cache_src.dtype != torch.bfloat16)
|
||||
output, lse = impl.forward_mqa(q.flatten(0, 1), kv_cache, metadata, layer)
|
||||
return metadata.decode.dcp_route.name, output, lse
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"heads_per_rank, qlen, block_size, fp8_kv, route",
|
||||
[
|
||||
(12, 6, 768, False, "TRITON"),
|
||||
(12, 6, 64, False, "SEGMENTED"),
|
||||
(16, 2, 16, True, "TRITON"),
|
||||
],
|
||||
ids=["kimi-k3-hybrid-kda-blocks", "segmented-fits", "deepseek-v3-mtp-fp8-kv"],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_dcp_verify_matches_attention(
|
||||
fake_dcp_groups, heads_per_rank, qlen, block_size, fp8_kv, route
|
||||
):
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.workspace import (
|
||||
init_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
)
|
||||
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
if not is_workspace_manager_initialized():
|
||||
init_workspace_manager(device)
|
||||
torch.manual_seed(0)
|
||||
seq_len = CTX + qlen
|
||||
q = torch.randn(
|
||||
NUM_REQS,
|
||||
qlen,
|
||||
heads_per_rank * DCP,
|
||||
HEAD_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
kv = torch.randn(NUM_REQS, seq_len, HEAD_SIZE, device=device).clamp_(-4, 4)
|
||||
if fp8_kv:
|
||||
k_scale = FP8_KV_SCALE
|
||||
kv_cache_src = (kv / k_scale).to(current_platform.fp8_dtype())
|
||||
else:
|
||||
k_scale = 1.0
|
||||
kv_cache_src = kv.to(torch.bfloat16)
|
||||
kv_ref = kv_cache_src.float() * k_scale
|
||||
|
||||
config = _vllm_config(heads_per_rank, qlen, fp8_kv)
|
||||
layer_name = "model.layers.0.self_attn.attn"
|
||||
with set_current_vllm_config(config):
|
||||
config.compilation_config.static_forward_context[layer_name] = _ctx_layer()
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=kv_cache_src.dtype,
|
||||
cache_dtype_str="fp8" if fp8_kv else None,
|
||||
non_causal_multi_token_decode=False,
|
||||
)
|
||||
shards = [
|
||||
_run_rank(
|
||||
config,
|
||||
spec,
|
||||
layer_name,
|
||||
fake_dcp_groups,
|
||||
rank,
|
||||
q,
|
||||
kv_cache_src,
|
||||
k_scale,
|
||||
)
|
||||
for rank in range(DCP)
|
||||
]
|
||||
|
||||
assert {shard_route for shard_route, _, _ in shards} == {route}
|
||||
lses = torch.stack([lse.float() for _, _, lse in shards])
|
||||
weights = torch.exp(lses - lses.max(dim=0).values)
|
||||
merged = (
|
||||
weights.unsqueeze(-1) * torch.stack([out.float() for _, out, _ in shards])
|
||||
).sum(dim=0) / weights.sum(dim=0).unsqueeze(-1)
|
||||
|
||||
# Query token t of a request sees the context plus verify tokens 0..t.
|
||||
scores = torch.einsum("rthd,rld->rthl", q.float(), kv_ref) * SCALE
|
||||
visible = torch.arange(seq_len, device=device) <= CTX + torch.arange(
|
||||
qlen, device=device
|
||||
).view(-1, 1)
|
||||
scores.masked_fill_(~visible[None, :, None, :], float("-inf"))
|
||||
reference = torch.einsum(
|
||||
"rthl,rld->rthd", scores.softmax(dim=-1), kv_ref[..., :KV_LORA_RANK]
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
merged.view_as(reference), reference, rtol=2e-2, atol=2e-2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"fp8_kv, num_heads, block_size",
|
||||
[
|
||||
(False, 96, 768),
|
||||
(False, 128, 64),
|
||||
(True, 64, 128),
|
||||
(True, 128, 16),
|
||||
],
|
||||
)
|
||||
def test_segmented_lds_gate_matches_the_kernel(fp8_kv, num_heads, block_size):
|
||||
"""The gate must flag exactly the shapes AITER segmented MLA cannot compile."""
|
||||
from triton.runtime.errors import OutOfResources
|
||||
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
_get_segmented_mla_decode,
|
||||
_segmented_mla_fits_lds,
|
||||
_segmented_mla_page_size,
|
||||
)
|
||||
|
||||
page = _segmented_mla_page_size(block_size)
|
||||
kv_dtype = current_platform.fp8_dtype() if fp8_kv else torch.bfloat16
|
||||
q = torch.zeros(1, num_heads, HEAD_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
kv = torch.zeros(1, page, 1, HEAD_SIZE, dtype=kv_dtype, device="cuda")
|
||||
one_row = torch.tensor([0, 1], dtype=torch.int32, device="cuda")
|
||||
try:
|
||||
_get_segmented_mla_decode()(
|
||||
q,
|
||||
kv,
|
||||
None,
|
||||
one_row,
|
||||
torch.tensor([page], dtype=torch.int32, device="cuda"),
|
||||
page,
|
||||
torch.zeros(1, 1, dtype=torch.int32, device="cuda"),
|
||||
SCALE,
|
||||
KV_LORA_RANK,
|
||||
ROPE_DIM,
|
||||
causal=True,
|
||||
q_descale=None,
|
||||
kv_descale=torch.tensor(1.0, device="cuda"),
|
||||
skip_reduce=True,
|
||||
)
|
||||
compiles = True
|
||||
except OutOfResources:
|
||||
compiles = False
|
||||
assert _segmented_mla_fits_lds(num_heads, block_size, fp8_kv) == compiles
|
||||
@@ -20,6 +20,10 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( # noqa: E402
|
||||
AiterMLAImpl,
|
||||
AiterMLAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.ops.dcp import ( # noqa: E402
|
||||
CPTritonContext,
|
||||
correct_attn_out,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_merge import ( # noqa: E402
|
||||
merge_mla_segments_triton,
|
||||
)
|
||||
@@ -121,8 +125,11 @@ def _builder(
|
||||
# segmented route is available, query length decides per batch. CPRR
|
||||
# preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back.
|
||||
_supports_segmented_dcp_verify=supports_segmented_dcp_verify,
|
||||
_use_triton_dcp_verify=False,
|
||||
# Derived once in the real constructor, so derive it once here too.
|
||||
_segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size),
|
||||
_dcp_verify_page_size=rocm_aiter_mla._segmented_mla_page_size(
|
||||
kernel_block_size
|
||||
),
|
||||
_dcp_verify_buffers=None,
|
||||
_graph_seq_lens=None,
|
||||
_kv_cache_dtype_str=kv_cache_dtype,
|
||||
@@ -630,6 +637,99 @@ def test_segmented_verify_reduce_returns_natural_lse_and_masks_empty_rows():
|
||||
assert lse[1].item() == float("-inf")
|
||||
|
||||
|
||||
def test_triton_verify_empty_local_shard_merges_through_dcp_combine():
|
||||
"""Empty rows need no masking: -inf LSE gives them zero combine weight.
|
||||
|
||||
Row 0 has KV on both ranks, row 1 only on rank 0, row 2 on neither (graph
|
||||
padding). The raw rows come back NaN; the production AG+RS combine must
|
||||
still produce full-context attention and zeros for the padded row.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_ranks, num_rows, num_heads = 2, 3, 16
|
||||
kv_lora_rank, head_dim, block_size = 512, 576, 16
|
||||
scale = head_dim**-0.5
|
||||
local_lens = [[5, 4, 0], [5, 0, 0]]
|
||||
|
||||
q = torch.randn(num_rows, num_heads, head_dim, dtype=torch.bfloat16, device="cuda")
|
||||
kv_caches = [
|
||||
torch.randn(1, block_size, head_dim, dtype=torch.bfloat16, device="cuda")
|
||||
for _ in range(num_ranks)
|
||||
]
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.scale = scale
|
||||
impl.kv_lora_rank = kv_lora_rank
|
||||
impl._sm_count = current_platform.num_compute_units()
|
||||
layer = SimpleNamespace(_k_scale=torch.tensor(1.0, device="cuda"))
|
||||
outputs, lses = [], []
|
||||
for rank in range(num_ranks):
|
||||
verify = SimpleNamespace(
|
||||
block_table=torch.zeros(num_rows, 1, dtype=torch.int32, device="cuda"),
|
||||
row_lens=torch.tensor(local_lens[rank], dtype=torch.int32, device="cuda"),
|
||||
max_kv_seq_len=max(local_lens[rank]),
|
||||
)
|
||||
output, lse = impl._forward_triton_dcp_verify(
|
||||
q, verify, kv_caches[rank], layer, torch.bfloat16
|
||||
)
|
||||
outputs.append(output)
|
||||
lses.append(lse)
|
||||
|
||||
assert torch.isneginf(lses[1][1]).all()
|
||||
assert torch.isneginf(lses[0][2]).all() and torch.isneginf(lses[1][2]).all()
|
||||
|
||||
all_lses = torch.stack(lses)
|
||||
merged = sum(
|
||||
correct_attn_out(outputs[rank], all_lses, rank, CPTritonContext())[0].float()
|
||||
for rank in range(num_ranks)
|
||||
)
|
||||
|
||||
reference = torch.zeros(num_rows, num_heads, kv_lora_rank, device="cuda")
|
||||
for row in range(num_rows):
|
||||
keys = torch.cat(
|
||||
[kv_caches[rank][0, : local_lens[rank][row]] for rank in range(num_ranks)]
|
||||
).float()
|
||||
if keys.shape[0]:
|
||||
probs = torch.softmax(q[row].float() @ keys.T * scale, dim=-1)
|
||||
reference[row] = probs @ keys[:, :kv_lora_rank]
|
||||
|
||||
assert torch.isfinite(merged).all()
|
||||
torch.testing.assert_close(merged, reference, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
def test_batch_aware_kv_splits_fit_an_int32_safe_reservation(monkeypatch):
|
||||
# MI325X DCP8 verify at 128K context: 128 gathered heads, 16K local tokens.
|
||||
sm_count, heads, local_seq_len, kv_lora_rank = 304, 128, 16384, 512
|
||||
max_rows = 4096
|
||||
reserved = []
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "is_workspace_manager_initialized", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"current_workspace_manager",
|
||||
lambda: SimpleNamespace(get_simultaneous=lambda *specs: reserved.extend(specs)),
|
||||
)
|
||||
rocm_aiter_mla._reserve_triton_dcp_verify_workspace(
|
||||
max_rows, heads, local_seq_len, kv_lora_rank, sm_count
|
||||
)
|
||||
((shape, _),) = reserved
|
||||
# The kernel's attn_logits offsets are int32; seq-only splits overflow them.
|
||||
seq_only = rocm_aiter_mla._compute_num_kv_splits(local_seq_len, sm_count)
|
||||
assert max_rows * heads * seq_only * (kv_lora_rank + 1) >= 2**31
|
||||
assert math.prod(shape) < 2**31
|
||||
|
||||
counts = [
|
||||
rocm_aiter_mla._triton_dcp_verify_num_kv_splits(
|
||||
local_seq_len, rows, heads, sm_count
|
||||
)
|
||||
for rows in range(1, max_rows + 1)
|
||||
]
|
||||
# Small batches keep the seq-only split count.
|
||||
assert counts[:4] == [seq_only] * 4
|
||||
assert counts == sorted(counts, reverse=True)
|
||||
for rows, splits in enumerate(counts, start=1):
|
||||
assert rows * heads * splits * (kv_lora_rank + 1) <= math.prod(shape)
|
||||
|
||||
|
||||
def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
|
||||
"""Exercise the target qlen>1 path and merge two rank-local results."""
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
|
||||
accumulate_mla_context_chunk,
|
||||
init_mla_context_partial,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.math_utils import cdiv, largest_power_of_2_divisor
|
||||
@@ -35,6 +36,7 @@ from vllm.v1.attention.backend import (
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.triton_mla import _compute_num_kv_splits
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
get_dcp_local_seq_lens,
|
||||
)
|
||||
@@ -46,7 +48,12 @@ from vllm.v1.attention.ops.rocm_aiter_mla_prefill import (
|
||||
expand_context,
|
||||
gather_compressed_context,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
|
||||
from vllm.v1.worker.workspace import (
|
||||
current_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
@@ -269,6 +276,75 @@ def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) ->
|
||||
)
|
||||
|
||||
|
||||
def _segmented_mla_fits_lds(num_heads: int, block_size: int, fp8_kv: bool) -> bool:
|
||||
"""Whether AITER segmented MLA compiles for this DCP verify shape.
|
||||
|
||||
On gfx942 it needs 128 KiB of LDS, over the 64 KiB available, for bf16 KV
|
||||
at TILE-128 and for fp8 KV past 64 heads, where its query block widens to
|
||||
128 heads. ``num_heads`` is the DCP-gathered count.
|
||||
"""
|
||||
from vllm.platforms.rocm import on_gfx942
|
||||
|
||||
if not on_gfx942():
|
||||
return True
|
||||
if fp8_kv:
|
||||
return num_heads <= 64
|
||||
return _segmented_mla_page_size(block_size) < 128
|
||||
|
||||
|
||||
def _triton_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) -> bool:
|
||||
"""Whether gfx942 can fall back to generic split-KV causal target verify.
|
||||
|
||||
The batch shape is causal qlen>1 over a DCP shard. Eagle MTP and DSpark
|
||||
target verify are that same shape, so this is not limited to one
|
||||
speculative method. gfx942 has no cprr kernel, so this serves the shapes
|
||||
segmented MLA cannot (see _segmented_mla_fits_lds). qlen==1 and non-causal
|
||||
stay on the plain ASM decode.
|
||||
"""
|
||||
from vllm.platforms.rocm import on_gfx942
|
||||
|
||||
return on_gfx942() and dcp_world_size > 1 and cp_interleave == 1
|
||||
|
||||
|
||||
# Heads per stage-1 program in the grouped MLA decode kernel (its BLOCK_H).
|
||||
_DECODE_HEADS_PER_PROGRAM: Final = 16
|
||||
# Tuned on MI325X with microbenchmarks
|
||||
_BATCH_OCCUPANCY_MULTIPLIER: Final = 4
|
||||
|
||||
|
||||
def _triton_dcp_verify_num_kv_splits(
|
||||
max_seq_len: int, num_rows: int, num_heads: int, sm_count: int
|
||||
) -> int:
|
||||
# Split only until the stage-1 grid fills the GPU. Past that, splits add
|
||||
# scratch and combine work, and large batches overflow the kernel's int32
|
||||
# attn_logits offsets.
|
||||
programs_per_split = max(1, num_rows) * cdiv(num_heads, _DECODE_HEADS_PER_PROGRAM)
|
||||
occupancy_splits = triton.next_power_of_2(
|
||||
cdiv(_BATCH_OCCUPANCY_MULTIPLIER * sm_count, programs_per_split)
|
||||
)
|
||||
return min(_compute_num_kv_splits(max_seq_len, sm_count), occupancy_splits)
|
||||
|
||||
|
||||
def _reserve_triton_dcp_verify_workspace(
|
||||
max_rows: int,
|
||||
num_heads: int,
|
||||
max_seq_len: int,
|
||||
kv_lora_rank: int,
|
||||
sm_count: int,
|
||||
) -> None:
|
||||
"""Reserve split-KV scratch before warmup locks the workspace manager."""
|
||||
if not is_workspace_manager_initialized():
|
||||
return
|
||||
# Fewer rows may take more splits, so size for the largest product.
|
||||
max_row_splits = max(
|
||||
rows * _triton_dcp_verify_num_kv_splits(max_seq_len, rows, num_heads, sm_count)
|
||||
for rows in range(1, max_rows + 1)
|
||||
)
|
||||
current_workspace_manager().get_simultaneous(
|
||||
((max_row_splits, num_heads, kv_lora_rank + 1), torch.float32),
|
||||
)
|
||||
|
||||
|
||||
# DCP decode routing
|
||||
#
|
||||
# With decode context parallelism each rank holds a round-robin shard of the KV
|
||||
@@ -281,9 +357,12 @@ def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) ->
|
||||
# round-robin ASM decode rebuilds global positions from g_kv_indptr, cp_rank
|
||||
# and cp_world_size and applies the causal window in-kernel. Kernels exist only
|
||||
# at _NATIVE_CPRR_HEADS; other gathered head counts are padded up.
|
||||
# * triton: causal qlen > 1 on gfx942 for shapes segmented MLA cannot fit in
|
||||
# LDS (_segmented_mla_fits_lds). Generic split-KV decode over the same
|
||||
# per-row local window segmented uses.
|
||||
# * segmented: causal qlen > 1 when cprr is not selected or qlen is below
|
||||
# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV length
|
||||
# encodes its causal window.
|
||||
# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV
|
||||
# length encodes its causal window.
|
||||
#
|
||||
# The plain kernel must never serve a causal qlen > 1 block: it applies
|
||||
# causality on local indices, which is wrong over a round-robin shard.
|
||||
@@ -295,6 +374,12 @@ class _DCPDecodeRoute(Enum):
|
||||
PLAIN = "plain"
|
||||
SEGMENTED = "segmented"
|
||||
CPRR = "cprr"
|
||||
TRITON = "triton"
|
||||
|
||||
|
||||
def _is_row_view_dcp_route(route: _DCPDecodeRoute) -> bool:
|
||||
"""Routes that attend one causal window per verify token."""
|
||||
return route in (_DCPDecodeRoute.SEGMENTED, _DCPDecodeRoute.TRITON)
|
||||
|
||||
|
||||
def _asm_dcp_verify_heads(decode_num_heads: int) -> int:
|
||||
@@ -308,7 +393,9 @@ def _asm_dcp_verify_heads(decode_num_heads: int) -> int:
|
||||
|
||||
|
||||
def _select_dcp_decode_route(
|
||||
*,
|
||||
supports_segmented: bool,
|
||||
supports_triton: bool,
|
||||
causal: bool,
|
||||
max_qo_len: int,
|
||||
asm_selected: bool,
|
||||
@@ -318,11 +405,13 @@ def _select_dcp_decode_route(
|
||||
return _DCPDecodeRoute.PLAIN
|
||||
if asm_selected and max_qo_len >= _MIN_CPRR_QLEN:
|
||||
return _DCPDecodeRoute.CPRR
|
||||
if supports_triton:
|
||||
return _DCPDecodeRoute.TRITON
|
||||
if supports_segmented:
|
||||
return _DCPDecodeRoute.SEGMENTED
|
||||
raise RuntimeError(
|
||||
"ROCM_AITER_MLA DCP multi-token verify requires either segmented "
|
||||
"MLA or the round-robin asm decode."
|
||||
"MLA, the round-robin asm decode, or the gfx942 Triton verify path."
|
||||
)
|
||||
|
||||
|
||||
@@ -472,11 +561,10 @@ class AiterMLABackend(MLACommonBackend):
|
||||
|
||||
@dataclass
|
||||
class AiterMLADCPVerifyMetadata:
|
||||
"""One paged-KV row per verify token, for segmented DCP verification.
|
||||
"""One paged-KV row per verify token for causal DCP verification.
|
||||
|
||||
These are produced together or not at all, so they travel as one value: its
|
||||
presence on the decode metadata *is* the routing decision the builder made,
|
||||
and the impl does not re-derive it.
|
||||
Segmented MLA and the gfx942 Triton path both consume this view. Which
|
||||
kernel runs is ``AiterMLADecodeMetadata.dcp_route``.
|
||||
"""
|
||||
|
||||
# Local KV length of each verify row, already causally bounded.
|
||||
@@ -514,7 +602,8 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
|
||||
min_kv_seq_len: int = 1
|
||||
# Exactly one DCP decode route is selected by the metadata builder.
|
||||
dcp_route: _DCPDecodeRoute = _DCPDecodeRoute.PLAIN
|
||||
# Set exactly when this batch routes to segmented DCP verification.
|
||||
# Set when this batch takes a per-row DCP verify kernel (segmented MLA or
|
||||
# gfx942 Triton). ``dcp_route`` selects which kernel.
|
||||
dcp_verify: AiterMLADCPVerifyMetadata | None = None
|
||||
# cprr only: global per-request page indptr and this rank's position.
|
||||
g_kv_indptr: torch.Tensor | None = None
|
||||
@@ -643,6 +732,10 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
parallel_config.decode_context_parallel_size,
|
||||
parallel_config.cp_kv_cache_interleave_size,
|
||||
)
|
||||
supports_triton_dcp_verify = _triton_dcp_verify_supported(
|
||||
parallel_config.decode_context_parallel_size,
|
||||
parallel_config.cp_kv_cache_interleave_size,
|
||||
)
|
||||
verify_route = envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY.lower()
|
||||
asm_dcp_verify_config = (
|
||||
_asm_dcp_verify_configured(
|
||||
@@ -659,7 +752,9 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
device,
|
||||
AiterMLAMetadata,
|
||||
supports_dcp_with_varlen=(
|
||||
supports_segmented_dcp_verify or asm_dcp_verify_config
|
||||
supports_segmented_dcp_verify
|
||||
or asm_dcp_verify_config
|
||||
or supports_triton_dcp_verify
|
||||
),
|
||||
)
|
||||
self._asm_dcp_verify = False
|
||||
@@ -684,7 +779,17 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
else "This AITER build also lacks segmented MLA decode."
|
||||
)
|
||||
)
|
||||
self._supports_segmented_dcp_verify = supports_segmented_dcp_verify
|
||||
self._supports_segmented_dcp_verify = (
|
||||
supports_segmented_dcp_verify
|
||||
and _segmented_mla_fits_lds(
|
||||
self.num_heads * self.dcp_world_size,
|
||||
kv_cache_spec.block_size,
|
||||
is_quantized_kv_cache(vllm_config.cache_config.cache_dtype),
|
||||
)
|
||||
)
|
||||
self._use_triton_dcp_verify = (
|
||||
supports_triton_dcp_verify and not self._supports_segmented_dcp_verify
|
||||
)
|
||||
self._mla_max_split_per_batch = 0
|
||||
if self._asm_dcp_verify:
|
||||
# Cap on KV splits per batch; at low batch the KV axis is the only
|
||||
@@ -723,7 +828,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
# (e.g. 16 with Eagle3), we expand block-level indices into per-token
|
||||
# flat indices since the aiter kernel always uses page_size=1 internally.
|
||||
self.kernel_block_size = kv_cache_spec.block_size
|
||||
self._segmented_page_size = _segmented_mla_page_size(self.kernel_block_size)
|
||||
# Generic Triton indexes physical KV blocks; segmented MLA indexes
|
||||
# TILE subpages of them.
|
||||
self._dcp_verify_page_size = (
|
||||
kv_cache_spec.block_size
|
||||
if self._use_triton_dcp_verify
|
||||
else _segmented_mla_page_size(kv_cache_spec.block_size)
|
||||
)
|
||||
# A DCP rank's shard of the longest sequence bounds every verify row.
|
||||
dcp_local_max_seq_len = (
|
||||
cdiv(
|
||||
vllm_config.model_config.max_model_len,
|
||||
self.dcp_world_size * parallel_config.cp_kv_cache_interleave_size,
|
||||
)
|
||||
* parallel_config.cp_kv_cache_interleave_size
|
||||
)
|
||||
|
||||
# In the flat view (.view(-1,1,1,H)), each token is its own page,
|
||||
# so max_num_pages_per_req = max_model_len regardless of
|
||||
@@ -886,21 +1005,16 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
max_num_reqs, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
if self._supports_segmented_dcp_verify and self._mtp_decode_qlen > 1:
|
||||
if (
|
||||
self._supports_segmented_dcp_verify or self._use_triton_dcp_verify
|
||||
) and self._mtp_decode_qlen > 1:
|
||||
# Allocate even when CPRR is the preferred route: a later
|
||||
# replay can fall below _MIN_CPRR_QLEN and needs these
|
||||
# addresses. A DCP rank's shard of the longest sequence
|
||||
# bounds every verify row, and full graphs need that bound
|
||||
# to be constant.
|
||||
num_dcp_partitions = (
|
||||
self.dcp_world_size * self.cp_kv_cache_interleave_size
|
||||
)
|
||||
graph_max_kv_seq_len = (
|
||||
cdiv(vllm_config.model_config.max_model_len, num_dcp_partitions)
|
||||
* self.cp_kv_cache_interleave_size
|
||||
)
|
||||
# addresses. Full graphs need the row bound to be constant.
|
||||
max_verify_rows = max_num_reqs * self._mtp_decode_qlen
|
||||
max_local_pages = cdiv(graph_max_kv_seq_len, self._segmented_page_size)
|
||||
max_local_pages = cdiv(
|
||||
dcp_local_max_seq_len, self._dcp_verify_page_size
|
||||
)
|
||||
self._dcp_verify_buffers = AiterMLADCPVerifyMetadata(
|
||||
row_lens=torch.zeros(
|
||||
max_verify_rows, dtype=torch.int32, device=device
|
||||
@@ -913,9 +1027,20 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
qo_indptr=torch.arange(
|
||||
max_verify_rows + 1, dtype=torch.int32, device=device
|
||||
),
|
||||
page_size=self._segmented_page_size,
|
||||
max_kv_seq_len=graph_max_kv_seq_len,
|
||||
page_size=self._dcp_verify_page_size,
|
||||
max_kv_seq_len=dcp_local_max_seq_len,
|
||||
)
|
||||
# gfx942 DCP verify launches the generic split-KV kernel, which draws
|
||||
# its partials from the shared workspace. Reserve it before warmup
|
||||
# locks the pool.
|
||||
if self._use_triton_dcp_verify and self._mtp_decode_qlen > 1:
|
||||
_reserve_triton_dcp_verify_workspace(
|
||||
max_num_reqs * self._mtp_decode_qlen,
|
||||
self._decode_num_heads,
|
||||
dcp_local_max_seq_len,
|
||||
self.mla_dims.kv_lora_rank,
|
||||
current_platform.num_compute_units(),
|
||||
)
|
||||
|
||||
def _init_fp8_prefill_ps_buffers(
|
||||
self,
|
||||
@@ -1193,7 +1318,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
block_table: torch.Tensor,
|
||||
dcp_tot_seq_lens: torch.Tensor,
|
||||
) -> AiterMLADCPVerifyMetadata:
|
||||
"""Build one paged-KV row per verify token for segmented DCP verification.
|
||||
"""Build one paged-KV row per verify token for DCP verification.
|
||||
|
||||
Every row is a single query (``qo_indptr`` is an arange), so the whole
|
||||
causal structure is carried by ``dcp_local_verify_row_lens`` and the
|
||||
@@ -1217,7 +1342,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
if buffers is not None
|
||||
else max(1, int(row_lens.max().item()))
|
||||
)
|
||||
page_size = self._segmented_page_size
|
||||
page_size = self._dcp_verify_page_size
|
||||
max_local_pages = cdiv(max_kv_seq_len, page_size)
|
||||
if buffers is not None:
|
||||
row_block_table = buffers.block_table[:num_rows]
|
||||
@@ -1235,7 +1360,9 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
dtype=torch.int32,
|
||||
device=block_table.device,
|
||||
)
|
||||
self._fill_dcp_verify_page_table(row_block_table, block_table, num_reqs, qlen)
|
||||
self._fill_dcp_verify_page_table(
|
||||
row_block_table, block_table, num_reqs, qlen, page_size
|
||||
)
|
||||
return AiterMLADCPVerifyMetadata(
|
||||
row_lens=row_lens,
|
||||
block_table=row_block_table,
|
||||
@@ -1250,8 +1377,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
block_table: torch.Tensor,
|
||||
num_reqs: int,
|
||||
qlen: int,
|
||||
page_size: int,
|
||||
) -> None:
|
||||
"""Expand each request's blocks into the subpages one verify row reads.
|
||||
"""Fill one page list per request and broadcast it across qlen rows.
|
||||
|
||||
``page_size`` is the unit the kernel indexes. Segmented MLA uses TILE
|
||||
subpages; generic Triton uses the physical KV block. When those are
|
||||
equal, pages_per_block is 1 and this copies block IDs as-is.
|
||||
|
||||
Every row of a request shares the request's shard, so the page list is
|
||||
built once per request and repeated; only ``row_lens`` distinguishes the
|
||||
@@ -1259,9 +1391,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
request's block table is always wider than the pages a row can reach.
|
||||
"""
|
||||
kernel_block_size = self.kernel_block_size
|
||||
page_size = self._segmented_page_size
|
||||
assert kernel_block_size is not None
|
||||
assert page_size is not None
|
||||
assert page_size > 0
|
||||
assert kernel_block_size % page_size == 0, (
|
||||
f"DCP verify page_size {page_size} must divide kernel_block_size "
|
||||
f"{kernel_block_size}"
|
||||
)
|
||||
pages_per_block = kernel_block_size // page_size
|
||||
max_local_pages = row_block_table.shape[1]
|
||||
max_local_blocks = cdiv(max_local_pages, pages_per_block)
|
||||
@@ -1269,15 +1404,18 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
f"DCP verify needs {max_local_blocks} blocks per request but the "
|
||||
f"block table only has {block_table.shape[1]}"
|
||||
)
|
||||
subpage_offsets = torch.arange(
|
||||
pages_per_block,
|
||||
dtype=block_table.dtype,
|
||||
device=block_table.device,
|
||||
)
|
||||
per_req_page_table = (
|
||||
block_table[:num_reqs, :max_local_blocks, None] * pages_per_block
|
||||
+ subpage_offsets
|
||||
).flatten(1)[:, :max_local_pages]
|
||||
if pages_per_block == 1:
|
||||
per_req_page_table = block_table[:num_reqs, :max_local_pages]
|
||||
else:
|
||||
subpage_offsets = torch.arange(
|
||||
pages_per_block,
|
||||
dtype=block_table.dtype,
|
||||
device=block_table.device,
|
||||
)
|
||||
per_req_page_table = (
|
||||
block_table[:num_reqs, :max_local_blocks, None] * pages_per_block
|
||||
+ subpage_offsets
|
||||
).flatten(1)[:, :max_local_pages]
|
||||
# Broadcast the request's page list across its rows instead of
|
||||
# materializing a repeat_interleave copy.
|
||||
row_block_table.unflatten(0, (num_reqs, qlen)).copy_(
|
||||
@@ -1373,18 +1511,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
dcp_route = _DCPDecodeRoute.PLAIN
|
||||
if self.dcp_world_size > 1:
|
||||
dcp_route = _select_dcp_decode_route(
|
||||
self._supports_segmented_dcp_verify,
|
||||
causal,
|
||||
int(max_qo_len),
|
||||
self._asm_dcp_verify,
|
||||
supports_segmented=self._supports_segmented_dcp_verify,
|
||||
supports_triton=self._use_triton_dcp_verify,
|
||||
causal=causal,
|
||||
max_qo_len=int(max_qo_len),
|
||||
asm_selected=self._asm_dcp_verify,
|
||||
)
|
||||
|
||||
# Segmented DCP verify carries its own per-row subpage table, so the
|
||||
# Row-view DCP verify carries its own per-row page table, so the
|
||||
# flat per-token view is dead work for it. Leave the buffer alone and
|
||||
# hand the metadata None, so a future reader cannot pick up whatever
|
||||
# the previous batch left behind.
|
||||
paged_kv_indices = None
|
||||
if dcp_route is not _DCPDecodeRoute.SEGMENTED:
|
||||
if not _is_row_view_dcp_route(dcp_route):
|
||||
if self.compilation_config.cudagraph_mode.has_full_cudagraphs():
|
||||
self.paged_kv_indices.fill_(-1)
|
||||
|
||||
@@ -1461,13 +1600,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
has_persistent_metadata = False
|
||||
# Only the asm decode consumes the schedule, so gate on the routing
|
||||
# rather than on num_heads >= 16, which denies it to a padded rank
|
||||
# running the same asm kernels. The predicates are disjoint -- decode
|
||||
# is qlen==1, verify is qlen>1 -- and cover both Gluon entries plus the
|
||||
# segmented DCP verify.
|
||||
# running the same asm kernels. Gluon and the row-view DCP routes
|
||||
# (segmented and gfx942 Triton) cover the non-ASM decode/verify paths.
|
||||
use_persistent_metadata = (
|
||||
not use_gluon_decode
|
||||
and not use_gluon_verify
|
||||
and dcp_route is not _DCPDecodeRoute.SEGMENTED
|
||||
and not _is_row_view_dcp_route(dcp_route)
|
||||
# A padded rank has no bf16 persistent kernel past qlen 4 where the
|
||||
# gfx950 fold is absent; the non-persistent entry covers it. fp8
|
||||
# keeps the schedule -- its fold rejects non-persistent outright.
|
||||
@@ -1566,7 +1704,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
min_kv_seq_len = int(per_req_len.min().item())
|
||||
|
||||
dcp_verify = None
|
||||
if dcp_route is _DCPDecodeRoute.SEGMENTED:
|
||||
if _is_row_view_dcp_route(dcp_route):
|
||||
assert causal
|
||||
assert dcp_tot_seq_lens_device is not None
|
||||
dcp_verify = self._build_dcp_verify_row_view(
|
||||
int(max_qo_len),
|
||||
@@ -2000,6 +2139,7 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
)
|
||||
AiterMLAHelper.check_num_heads_validity(num_heads)
|
||||
AiterMLAHelper.check_num_heads_validity(self._decode_num_heads)
|
||||
self._sm_count = current_platform.num_compute_units()
|
||||
|
||||
unsupported_features = [alibi_slopes, sliding_window, logits_soft_cap]
|
||||
if any(unsupported_features):
|
||||
@@ -2389,6 +2529,64 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
out_dtype,
|
||||
)
|
||||
|
||||
def _forward_triton_dcp_verify(
|
||||
self,
|
||||
q_mla: torch.Tensor,
|
||||
verify: AiterMLADCPVerifyMetadata,
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
layer: AttentionLayer,
|
||||
out_dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""gfx942 causal verify using the LDS-safe generic split-KV kernel.
|
||||
|
||||
Each verify token is its own row whose causal window is its
|
||||
``row_lens`` entry. A zero-length row (an empty DCP shard or graph
|
||||
padding) comes back as a NaN output with an LSE of -inf; the DCP
|
||||
combine gives it zero weight and never reads the output.
|
||||
"""
|
||||
num_rows, num_heads = q_mla.shape[:2]
|
||||
# Stage 2 stores every output and LSE element, empty rows included.
|
||||
output = torch.empty(
|
||||
num_rows, num_heads, self.kv_lora_rank, dtype=out_dtype, device=q_mla.device
|
||||
)
|
||||
# Matches merge_mla_segments_triton and the fp32 LSE the DCP combine
|
||||
# kernel is warmed up for.
|
||||
lse = torch.empty(num_rows, num_heads, dtype=torch.float32, device=q_mla.device)
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
num_kv_splits = 1
|
||||
else:
|
||||
num_kv_splits = _triton_dcp_verify_num_kv_splits(
|
||||
verify.max_kv_seq_len, num_rows, num_heads, self._sm_count
|
||||
)
|
||||
logits_shape = (num_rows, num_heads, num_kv_splits, self.kv_lora_rank + 1)
|
||||
if is_workspace_manager_initialized():
|
||||
(attn_logits,) = current_workspace_manager().get_simultaneous(
|
||||
(logits_shape, torch.float32),
|
||||
)
|
||||
else:
|
||||
attn_logits = torch.empty(
|
||||
logits_shape, dtype=torch.float32, device=q_mla.device
|
||||
)
|
||||
|
||||
paged_kv = kv_c_and_k_pe_cache.unsqueeze(2)
|
||||
decode_attention_fwd(
|
||||
q_mla,
|
||||
paged_kv,
|
||||
paged_kv[..., : self.kv_lora_rank],
|
||||
output,
|
||||
lse,
|
||||
verify.block_table,
|
||||
verify.row_lens,
|
||||
attn_logits,
|
||||
num_kv_splits,
|
||||
self.scale,
|
||||
paged_kv.size(1),
|
||||
k_scale=layer._k_scale,
|
||||
v_scale=layer._k_scale,
|
||||
is_mla=True,
|
||||
)
|
||||
return output, lse
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
@@ -2499,6 +2697,23 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
return o, None
|
||||
|
||||
dcp_route = decode.dcp_route
|
||||
if dcp_route is _DCPDecodeRoute.TRITON:
|
||||
verify = decode.dcp_verify
|
||||
assert verify is not None
|
||||
q_mla = torch.cat(q, dim=-1) if isinstance(q, tuple) else q
|
||||
if (
|
||||
is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
and q_mla.dtype != torch.bfloat16
|
||||
):
|
||||
q_mla = q_mla.to(torch.bfloat16) * layer._q_scale
|
||||
return self._forward_triton_dcp_verify(
|
||||
q_mla,
|
||||
verify,
|
||||
kv_c_and_k_pe_cache,
|
||||
layer,
|
||||
decode.attn_out_dtype,
|
||||
)
|
||||
|
||||
if dcp_route is _DCPDecodeRoute.SEGMENTED:
|
||||
verify = decode.dcp_verify
|
||||
assert verify is not None
|
||||
|
||||
Reference in New Issue
Block a user