[ROCm][MLA] Route gfx942 DCP causal multi-token verify to Triton split-KV decode (#58555)
Signed-off-by: Xavier Aguilar <xavier.aguilarfruto@amd.com>
This commit is contained in:
@@ -161,6 +161,11 @@ def _build(monkeypatch, route, gathered_heads, segmented_supported=True):
|
||||
lambda *args: segmented_supported,
|
||||
)
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_fp8_mla_prefill_supported", lambda: False)
|
||||
# These cases pick between cprr and segmented. On a gfx942 runner the real
|
||||
# arch would add the Triton verify route, which this stub cannot build.
|
||||
import vllm.platforms.rocm as rocm
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx942", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "get_dcp_group", lambda: SimpleNamespace(rank_in_group=0)
|
||||
)
|
||||
@@ -264,25 +269,46 @@ def test_min_cprr_qlen_is_above_two():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"supports,causal,qlen,asm,expected",
|
||||
"segmented,triton,causal,qlen,asm,expected",
|
||||
[
|
||||
(True, True, 1, True, Route.PLAIN),
|
||||
(True, True, 2, True, Route.SEGMENTED),
|
||||
(True, True, 3, True, Route.CPRR),
|
||||
(True, True, 4, True, Route.CPRR),
|
||||
(True, True, 4, False, Route.SEGMENTED),
|
||||
(True, False, 2, True, Route.PLAIN),
|
||||
(True, False, 4, False, Route.PLAIN),
|
||||
(False, True, 3, True, Route.CPRR),
|
||||
(True, False, True, 1, True, Route.PLAIN),
|
||||
(True, False, True, 2, True, Route.SEGMENTED),
|
||||
(True, False, True, 3, True, Route.CPRR),
|
||||
(True, False, True, 4, True, Route.CPRR),
|
||||
(True, False, True, 4, False, Route.SEGMENTED),
|
||||
(True, False, False, 2, True, Route.PLAIN),
|
||||
(True, False, False, 4, False, Route.PLAIN),
|
||||
(False, False, True, 3, True, Route.CPRR),
|
||||
# gfx942: no cprr and no segmented, so causal verify goes to Triton.
|
||||
(False, True, True, 1, False, Route.PLAIN),
|
||||
(False, True, True, 2, False, Route.TRITON),
|
||||
(False, True, True, 4, False, Route.TRITON),
|
||||
(False, True, False, 4, False, Route.PLAIN),
|
||||
# cprr still wins at or above its qlen floor.
|
||||
(False, True, True, 4, True, Route.CPRR),
|
||||
(False, True, True, 2, True, Route.TRITON),
|
||||
],
|
||||
)
|
||||
def test_dcp_decode_route(supports, causal, qlen, asm, expected):
|
||||
assert _select_route(supports, causal, qlen, asm) is expected
|
||||
def test_dcp_decode_route(segmented, triton, causal, qlen, asm, expected):
|
||||
route = _select_route(
|
||||
supports_segmented=segmented,
|
||||
supports_triton=triton,
|
||||
causal=causal,
|
||||
max_qo_len=qlen,
|
||||
asm_selected=asm,
|
||||
)
|
||||
assert route is expected
|
||||
|
||||
|
||||
def test_causal_multi_token_batch_without_a_valid_route_fails():
|
||||
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
|
||||
_select_route(False, True, 2, True)
|
||||
_select_route(
|
||||
supports_segmented=False,
|
||||
supports_triton=False,
|
||||
causal=True,
|
||||
max_qo_len=2,
|
||||
asm_selected=True,
|
||||
)
|
||||
|
||||
|
||||
# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs
|
||||
|
||||
@@ -536,5 +536,12 @@ def test_cprr_kernel_is_selected_for_the_k3_shape():
|
||||
is True
|
||||
)
|
||||
assert QLEN >= m._MIN_CPRR_QLEN
|
||||
assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR
|
||||
route = m._select_dcp_decode_route(
|
||||
supports_segmented=True,
|
||||
supports_triton=False,
|
||||
causal=True,
|
||||
max_qo_len=QLEN,
|
||||
asm_selected=True,
|
||||
)
|
||||
assert route is m._DCPDecodeRoute.CPRR
|
||||
assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Causal target verify under DCP on gfx942 (MI300/MI325).
|
||||
|
||||
Reproduces the attention shape of
|
||||
|
||||
vllm serve moonshotai/Kimi-K3 --tensor-parallel-size 8 \
|
||||
--decode-context-parallel-size 8 \
|
||||
--speculative-config '{"method": "dspark", "num_speculative_tokens": 5}'
|
||||
|
||||
96 gathered heads, bf16 KV, qlen 6 and the 768-token blocks the hybrid KDA
|
||||
layers impose. Segmented MLA tiles those blocks at 128 tokens and needs 128 KiB
|
||||
of LDS (gfx942 has 64 KiB), so the builder must route this shape to the generic
|
||||
Triton split-KV kernel. DeepSeek-V3 with MTP and an fp8 KV cache at DCP 8 hits
|
||||
the same limit through its 128 gathered heads. Shapes segmented MLA fits stay
|
||||
on it.
|
||||
|
||||
The real metadata builder and ``forward_mqa`` run on one GPU with the DCP ranks
|
||||
simulated one after another. That is sound because this path has no
|
||||
collective: the query all-gather happens in the layer, before ``forward_mqa``.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm AITER MLA tests", allow_module_level=True)
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found # noqa: E402
|
||||
|
||||
# DeepSeek-R1 supplies the MLA dims, which Kimi-K3 and DeepSeek-V3 share; only
|
||||
# the head count is overridden.
|
||||
MODEL = "deepseek-ai/DeepSeek-R1"
|
||||
KV_LORA_RANK, ROPE_DIM, NOPE_DIM = 512, 64, 128
|
||||
HEAD_SIZE = KV_LORA_RANK + ROPE_DIM
|
||||
SCALE = (NOPE_DIM + ROPE_DIM) ** -0.5
|
||||
DCP = 8
|
||||
# Each rank holds ~800 tokens per request, so its shard spans several blocks.
|
||||
NUM_REQS, CTX = 4, 6400
|
||||
FP8_KV_SCALE = 0.05
|
||||
|
||||
|
||||
def _on_gfx942() -> bool:
|
||||
if not (is_aiter_found() and torch.accelerator.is_available()):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx942
|
||||
|
||||
return on_gfx942()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(not _on_gfx942(), reason="needs ROCm + AITER on gfx942")
|
||||
|
||||
|
||||
class _FakeGroup:
|
||||
def __init__(self, world_size: int, rank: int):
|
||||
self.world_size = world_size
|
||||
self.rank_in_group = self.rank = self.local_rank = rank
|
||||
|
||||
def all_gather(self, tensor, dim=0):
|
||||
raise RuntimeError("DCP verify must not issue a collective")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_dcp_groups(monkeypatch):
|
||||
"""Point every importer of get_dcp_group at a settable rank stub."""
|
||||
import vllm.distributed.parallel_state as ps
|
||||
import vllm.model_executor.layers.attention.mla_attention as mla_attention
|
||||
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
|
||||
|
||||
group = _FakeGroup(DCP, 0)
|
||||
for module in (ps, mla_attention, backend):
|
||||
monkeypatch.setattr(module, "get_dcp_group", lambda: group)
|
||||
monkeypatch.setattr(ps, "get_tp_group", lambda: _FakeGroup(1, 0))
|
||||
return group
|
||||
|
||||
|
||||
def _vllm_config(heads_per_rank: int, qlen: int, fp8_kv: bool):
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
|
||||
config = create_vllm_config(
|
||||
model_name=MODEL,
|
||||
max_model_len=CTX + qlen,
|
||||
max_num_seqs=NUM_REQS,
|
||||
max_num_batched_tokens=4096,
|
||||
)
|
||||
config.model_config.model_arch_config.total_num_attention_heads = heads_per_rank
|
||||
# The reorder threshold reads only these two fields; without them a
|
||||
# multi-token row is classified as prefill and never reaches decode.
|
||||
config.speculative_config = SimpleNamespace(
|
||||
num_speculative_tokens=qlen - 1, parallel_drafting=False
|
||||
)
|
||||
config.parallel_config.decode_context_parallel_size = DCP
|
||||
config.parallel_config.cp_kv_cache_interleave_size = 1
|
||||
if fp8_kv:
|
||||
config.cache_config.cache_dtype = "fp8"
|
||||
return config
|
||||
|
||||
|
||||
def _ctx_layer():
|
||||
"""The registered-layer fields MLACommonMetadataBuilder.__init__ reads."""
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
|
||||
dcp_manager = object.__new__(MLADCPManager)
|
||||
dcp_manager.init_kv_gather = lambda *args, **kwargs: None
|
||||
# Only chunked-context prefill uses the cloned backend.
|
||||
prefill_backend = SimpleNamespace()
|
||||
prefill_backend.clone = lambda: prefill_backend
|
||||
return SimpleNamespace(
|
||||
non_causal_multi_token_decode=False,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=NOPE_DIM,
|
||||
qk_rope_head_dim=ROPE_DIM,
|
||||
v_head_dim=NOPE_DIM,
|
||||
dcp_manager=dcp_manager,
|
||||
prefill_backend=prefill_backend,
|
||||
)
|
||||
|
||||
|
||||
def _impl(dcp_rank: int, heads_per_rank: int, fp8_kv: bool):
|
||||
"""forward_mqa needs these attributes, not the weight-loading __init__."""
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.num_heads = heads_per_rank
|
||||
impl.kv_lora_rank = KV_LORA_RANK
|
||||
impl.qk_rope_head_dim = ROPE_DIM
|
||||
impl.dcp_world_size = DCP
|
||||
impl.dcp_rank = dcp_rank
|
||||
impl.pcp_world_size = 1
|
||||
impl.kv_cache_dtype = "fp8" if fp8_kv else "auto"
|
||||
impl.scale = SCALE
|
||||
impl._sm_count = current_platform.num_compute_units()
|
||||
return impl
|
||||
|
||||
|
||||
def _run_rank(config, spec, layer_name, group, rank, q, kv_cache_src, k_scale):
|
||||
"""Shard the KV round-robin onto ``rank`` and run its verify step."""
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
|
||||
|
||||
device = q.device
|
||||
qlen, heads = q.shape[1], q.shape[2] // DCP
|
||||
seq_len = CTX + qlen
|
||||
positions = torch.arange(rank, seq_len, DCP, device=device)
|
||||
local_len = positions.numel()
|
||||
# Production block tables span max_model_len, so every rank's table is at
|
||||
# least as wide as the largest shard needs.
|
||||
blocks_per_req = cdiv(cdiv(seq_len, DCP), spec.block_size)
|
||||
kv_cache = torch.zeros(
|
||||
NUM_REQS * blocks_per_req,
|
||||
spec.block_size,
|
||||
HEAD_SIZE,
|
||||
dtype=kv_cache_src.dtype,
|
||||
device=device,
|
||||
)
|
||||
kv_cache.view(NUM_REQS, -1, HEAD_SIZE)[:, :local_len] = kv_cache_src[:, positions]
|
||||
block_table = torch.arange(
|
||||
NUM_REQS * blocks_per_req, dtype=torch.int32, device=device
|
||||
).view(NUM_REQS, blocks_per_req)
|
||||
|
||||
query_start_loc = torch.arange(
|
||||
0, NUM_REQS * qlen + 1, qlen, dtype=torch.int32, device=device
|
||||
)
|
||||
local_lens = torch.full((NUM_REQS,), local_len, dtype=torch.int32)
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.full((NUM_REQS,), seq_len, dtype=torch.int32, device=device),
|
||||
num_reqs=NUM_REQS,
|
||||
num_actual_tokens=NUM_REQS * qlen,
|
||||
max_query_len=qlen,
|
||||
max_seq_len=seq_len,
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=torch.arange(NUM_REQS * qlen, dtype=torch.int64, device=device),
|
||||
causal=True,
|
||||
dcp_local_seq_lens=local_lens.to(device),
|
||||
dcp_local_seq_lens_cpu_upper_bound=local_lens,
|
||||
)
|
||||
|
||||
group.rank_in_group = group.rank = group.local_rank = rank
|
||||
builder = AiterMLAMetadataBuilder(spec, [layer_name], config, device)
|
||||
metadata = builder.build(0, common)
|
||||
layer = SimpleNamespace(
|
||||
_q_scale=torch.tensor(1.0, device=device),
|
||||
_k_scale=torch.tensor(k_scale, device=device),
|
||||
)
|
||||
impl = _impl(rank, heads, kv_cache_src.dtype != torch.bfloat16)
|
||||
output, lse = impl.forward_mqa(q.flatten(0, 1), kv_cache, metadata, layer)
|
||||
return metadata.decode.dcp_route.name, output, lse
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"heads_per_rank, qlen, block_size, fp8_kv, route",
|
||||
[
|
||||
(12, 6, 768, False, "TRITON"),
|
||||
(12, 6, 64, False, "SEGMENTED"),
|
||||
(16, 2, 16, True, "TRITON"),
|
||||
],
|
||||
ids=["kimi-k3-hybrid-kda-blocks", "segmented-fits", "deepseek-v3-mtp-fp8-kv"],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_dcp_verify_matches_attention(
|
||||
fake_dcp_groups, heads_per_rank, qlen, block_size, fp8_kv, route
|
||||
):
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.workspace import (
|
||||
init_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
)
|
||||
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
if not is_workspace_manager_initialized():
|
||||
init_workspace_manager(device)
|
||||
torch.manual_seed(0)
|
||||
seq_len = CTX + qlen
|
||||
q = torch.randn(
|
||||
NUM_REQS,
|
||||
qlen,
|
||||
heads_per_rank * DCP,
|
||||
HEAD_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
kv = torch.randn(NUM_REQS, seq_len, HEAD_SIZE, device=device).clamp_(-4, 4)
|
||||
if fp8_kv:
|
||||
k_scale = FP8_KV_SCALE
|
||||
kv_cache_src = (kv / k_scale).to(current_platform.fp8_dtype())
|
||||
else:
|
||||
k_scale = 1.0
|
||||
kv_cache_src = kv.to(torch.bfloat16)
|
||||
kv_ref = kv_cache_src.float() * k_scale
|
||||
|
||||
config = _vllm_config(heads_per_rank, qlen, fp8_kv)
|
||||
layer_name = "model.layers.0.self_attn.attn"
|
||||
with set_current_vllm_config(config):
|
||||
config.compilation_config.static_forward_context[layer_name] = _ctx_layer()
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=kv_cache_src.dtype,
|
||||
cache_dtype_str="fp8" if fp8_kv else None,
|
||||
non_causal_multi_token_decode=False,
|
||||
)
|
||||
shards = [
|
||||
_run_rank(
|
||||
config,
|
||||
spec,
|
||||
layer_name,
|
||||
fake_dcp_groups,
|
||||
rank,
|
||||
q,
|
||||
kv_cache_src,
|
||||
k_scale,
|
||||
)
|
||||
for rank in range(DCP)
|
||||
]
|
||||
|
||||
assert {shard_route for shard_route, _, _ in shards} == {route}
|
||||
lses = torch.stack([lse.float() for _, _, lse in shards])
|
||||
weights = torch.exp(lses - lses.max(dim=0).values)
|
||||
merged = (
|
||||
weights.unsqueeze(-1) * torch.stack([out.float() for _, out, _ in shards])
|
||||
).sum(dim=0) / weights.sum(dim=0).unsqueeze(-1)
|
||||
|
||||
# Query token t of a request sees the context plus verify tokens 0..t.
|
||||
scores = torch.einsum("rthd,rld->rthl", q.float(), kv_ref) * SCALE
|
||||
visible = torch.arange(seq_len, device=device) <= CTX + torch.arange(
|
||||
qlen, device=device
|
||||
).view(-1, 1)
|
||||
scores.masked_fill_(~visible[None, :, None, :], float("-inf"))
|
||||
reference = torch.einsum(
|
||||
"rthl,rld->rthd", scores.softmax(dim=-1), kv_ref[..., :KV_LORA_RANK]
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
merged.view_as(reference), reference, rtol=2e-2, atol=2e-2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"fp8_kv, num_heads, block_size",
|
||||
[
|
||||
(False, 96, 768),
|
||||
(False, 128, 64),
|
||||
(True, 64, 128),
|
||||
(True, 128, 16),
|
||||
],
|
||||
)
|
||||
def test_segmented_lds_gate_matches_the_kernel(fp8_kv, num_heads, block_size):
|
||||
"""The gate must flag exactly the shapes AITER segmented MLA cannot compile."""
|
||||
from triton.runtime.errors import OutOfResources
|
||||
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
_get_segmented_mla_decode,
|
||||
_segmented_mla_fits_lds,
|
||||
_segmented_mla_page_size,
|
||||
)
|
||||
|
||||
page = _segmented_mla_page_size(block_size)
|
||||
kv_dtype = current_platform.fp8_dtype() if fp8_kv else torch.bfloat16
|
||||
q = torch.zeros(1, num_heads, HEAD_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
kv = torch.zeros(1, page, 1, HEAD_SIZE, dtype=kv_dtype, device="cuda")
|
||||
one_row = torch.tensor([0, 1], dtype=torch.int32, device="cuda")
|
||||
try:
|
||||
_get_segmented_mla_decode()(
|
||||
q,
|
||||
kv,
|
||||
None,
|
||||
one_row,
|
||||
torch.tensor([page], dtype=torch.int32, device="cuda"),
|
||||
page,
|
||||
torch.zeros(1, 1, dtype=torch.int32, device="cuda"),
|
||||
SCALE,
|
||||
KV_LORA_RANK,
|
||||
ROPE_DIM,
|
||||
causal=True,
|
||||
q_descale=None,
|
||||
kv_descale=torch.tensor(1.0, device="cuda"),
|
||||
skip_reduce=True,
|
||||
)
|
||||
compiles = True
|
||||
except OutOfResources:
|
||||
compiles = False
|
||||
assert _segmented_mla_fits_lds(num_heads, block_size, fp8_kv) == compiles
|
||||
@@ -20,6 +20,10 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( # noqa: E402
|
||||
AiterMLAImpl,
|
||||
AiterMLAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.ops.dcp import ( # noqa: E402
|
||||
CPTritonContext,
|
||||
correct_attn_out,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_merge import ( # noqa: E402
|
||||
merge_mla_segments_triton,
|
||||
)
|
||||
@@ -121,8 +125,11 @@ def _builder(
|
||||
# segmented route is available, query length decides per batch. CPRR
|
||||
# preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back.
|
||||
_supports_segmented_dcp_verify=supports_segmented_dcp_verify,
|
||||
_use_triton_dcp_verify=False,
|
||||
# Derived once in the real constructor, so derive it once here too.
|
||||
_segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size),
|
||||
_dcp_verify_page_size=rocm_aiter_mla._segmented_mla_page_size(
|
||||
kernel_block_size
|
||||
),
|
||||
_dcp_verify_buffers=None,
|
||||
_graph_seq_lens=None,
|
||||
_kv_cache_dtype_str=kv_cache_dtype,
|
||||
@@ -630,6 +637,99 @@ def test_segmented_verify_reduce_returns_natural_lse_and_masks_empty_rows():
|
||||
assert lse[1].item() == float("-inf")
|
||||
|
||||
|
||||
def test_triton_verify_empty_local_shard_merges_through_dcp_combine():
|
||||
"""Empty rows need no masking: -inf LSE gives them zero combine weight.
|
||||
|
||||
Row 0 has KV on both ranks, row 1 only on rank 0, row 2 on neither (graph
|
||||
padding). The raw rows come back NaN; the production AG+RS combine must
|
||||
still produce full-context attention and zeros for the padded row.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_ranks, num_rows, num_heads = 2, 3, 16
|
||||
kv_lora_rank, head_dim, block_size = 512, 576, 16
|
||||
scale = head_dim**-0.5
|
||||
local_lens = [[5, 4, 0], [5, 0, 0]]
|
||||
|
||||
q = torch.randn(num_rows, num_heads, head_dim, dtype=torch.bfloat16, device="cuda")
|
||||
kv_caches = [
|
||||
torch.randn(1, block_size, head_dim, dtype=torch.bfloat16, device="cuda")
|
||||
for _ in range(num_ranks)
|
||||
]
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.scale = scale
|
||||
impl.kv_lora_rank = kv_lora_rank
|
||||
impl._sm_count = current_platform.num_compute_units()
|
||||
layer = SimpleNamespace(_k_scale=torch.tensor(1.0, device="cuda"))
|
||||
outputs, lses = [], []
|
||||
for rank in range(num_ranks):
|
||||
verify = SimpleNamespace(
|
||||
block_table=torch.zeros(num_rows, 1, dtype=torch.int32, device="cuda"),
|
||||
row_lens=torch.tensor(local_lens[rank], dtype=torch.int32, device="cuda"),
|
||||
max_kv_seq_len=max(local_lens[rank]),
|
||||
)
|
||||
output, lse = impl._forward_triton_dcp_verify(
|
||||
q, verify, kv_caches[rank], layer, torch.bfloat16
|
||||
)
|
||||
outputs.append(output)
|
||||
lses.append(lse)
|
||||
|
||||
assert torch.isneginf(lses[1][1]).all()
|
||||
assert torch.isneginf(lses[0][2]).all() and torch.isneginf(lses[1][2]).all()
|
||||
|
||||
all_lses = torch.stack(lses)
|
||||
merged = sum(
|
||||
correct_attn_out(outputs[rank], all_lses, rank, CPTritonContext())[0].float()
|
||||
for rank in range(num_ranks)
|
||||
)
|
||||
|
||||
reference = torch.zeros(num_rows, num_heads, kv_lora_rank, device="cuda")
|
||||
for row in range(num_rows):
|
||||
keys = torch.cat(
|
||||
[kv_caches[rank][0, : local_lens[rank][row]] for rank in range(num_ranks)]
|
||||
).float()
|
||||
if keys.shape[0]:
|
||||
probs = torch.softmax(q[row].float() @ keys.T * scale, dim=-1)
|
||||
reference[row] = probs @ keys[:, :kv_lora_rank]
|
||||
|
||||
assert torch.isfinite(merged).all()
|
||||
torch.testing.assert_close(merged, reference, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
def test_batch_aware_kv_splits_fit_an_int32_safe_reservation(monkeypatch):
|
||||
# MI325X DCP8 verify at 128K context: 128 gathered heads, 16K local tokens.
|
||||
sm_count, heads, local_seq_len, kv_lora_rank = 304, 128, 16384, 512
|
||||
max_rows = 4096
|
||||
reserved = []
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "is_workspace_manager_initialized", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"current_workspace_manager",
|
||||
lambda: SimpleNamespace(get_simultaneous=lambda *specs: reserved.extend(specs)),
|
||||
)
|
||||
rocm_aiter_mla._reserve_triton_dcp_verify_workspace(
|
||||
max_rows, heads, local_seq_len, kv_lora_rank, sm_count
|
||||
)
|
||||
((shape, _),) = reserved
|
||||
# The kernel's attn_logits offsets are int32; seq-only splits overflow them.
|
||||
seq_only = rocm_aiter_mla._compute_num_kv_splits(local_seq_len, sm_count)
|
||||
assert max_rows * heads * seq_only * (kv_lora_rank + 1) >= 2**31
|
||||
assert math.prod(shape) < 2**31
|
||||
|
||||
counts = [
|
||||
rocm_aiter_mla._triton_dcp_verify_num_kv_splits(
|
||||
local_seq_len, rows, heads, sm_count
|
||||
)
|
||||
for rows in range(1, max_rows + 1)
|
||||
]
|
||||
# Small batches keep the seq-only split count.
|
||||
assert counts[:4] == [seq_only] * 4
|
||||
assert counts == sorted(counts, reverse=True)
|
||||
for rows, splits in enumerate(counts, start=1):
|
||||
assert rows * heads * splits * (kv_lora_rank + 1) <= math.prod(shape)
|
||||
|
||||
|
||||
def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
|
||||
"""Exercise the target qlen>1 path and merge two rank-local results."""
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
|
||||
|
||||
Reference in New Issue
Block a user