[ROCm][MLA] Reach FULL cudagraphs for AITER MLA speculative decoding (#51171)
Signed-off-by: yudigege86 <naqin@amd.com> Co-authored-by: yudigege86 <naqin@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
yudigege86
Cursor
Andreas Karatzas
parent
b5707bf994
commit
dbf662c9e8
@@ -1,21 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for causal masking in the AITER MLA verify flatten.
|
||||
"""Regression test for causal masking in the AITER MLA verify MTP path.
|
||||
|
||||
When ``num_heads < 16`` the ROCm AITER MLA backend cannot use the ASM decode
|
||||
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` flattens a
|
||||
``1 + num_speculative_tokens`` verify block into that many single-token Gluon
|
||||
decodes. Every resulting row used to be handed its request's entire paged-KV
|
||||
range. ``paged_kv_indptr`` is the cumulative sum of ``seq_lens``, which counts
|
||||
the tokens scheduled in the current step, so that range already spans the
|
||||
verify block: each verify position could attend to the draft tokens after it,
|
||||
which are the tokens it is supposed to be checking. Nothing crashes and nothing
|
||||
warns, so the only way to catch it is to look at the KV window per row.
|
||||
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` routes verify
|
||||
through ``mla_gluon``'s native 4-D MTP entry (``q`` shaped
|
||||
``[batch, qlen, nhead, dim]`` with ``use_2d_view=False``). aiter applies the
|
||||
causal tail internally: position ``t`` attends KV ``[0, seq_len - qlen + t]``.
|
||||
|
||||
The test drives the real builder and the real ``forward_mqa`` over a
|
||||
multi-token decode batch, intercepts the ``(page_table, seq_info)`` actually
|
||||
handed to the Gluon kernel, and pins verify row ``t`` of request ``r`` to the
|
||||
``context_r + t + 1`` entries of that request's ascending slice.
|
||||
The test drives the real builder and ``forward_mqa`` over a multi-token decode
|
||||
batch, intercepts the arguments actually handed to the Gluon kernel, and pins
|
||||
that the 4-D MTP contract is used with per-request ``block_table`` and
|
||||
``cache_seqlens`` rather than a flattened per-token paged-KV view.
|
||||
"""
|
||||
|
||||
import types
|
||||
@@ -32,15 +28,15 @@ def _on_rocm_with_aiter() -> bool:
|
||||
return current_platform.is_rocm() and is_aiter_found()
|
||||
|
||||
|
||||
# The production verify flatten is gfx950-only. This test replaces the Gluon
|
||||
# The production verify MTP path is gfx950-only. This test replaces the Gluon
|
||||
# kernel with a spy and forces the architecture feature probe so the metadata
|
||||
# regression remains testable on every ROCm AITER CI runner.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _on_rocm_with_aiter(),
|
||||
reason="ROCM_AITER_MLA verify flatten requires ROCm and AITER",
|
||||
reason="ROCM_AITER_MLA verify MTP requires ROCm and AITER",
|
||||
)
|
||||
|
||||
# Below the 16-head threshold that selects the flatten. DeepSeek-V3 / R1 at
|
||||
# Below the 16-head threshold that selects the MTP path. DeepSeek-V3 / R1 at
|
||||
# TP=16 and Kimi-K3 at TP=8 both land here.
|
||||
NUM_QUERY_HEADS = 12
|
||||
KV_LORA_RANK = 512
|
||||
@@ -54,8 +50,7 @@ QLEN = 8
|
||||
# Committed context per request, including a fresh request at zero context
|
||||
# where the whole KV range is the verify block itself.
|
||||
CONTEXT_LENS = [0, 1, 37, 512]
|
||||
# A cudagraph padding request: seq_len 0, so every one of its rows must clamp
|
||||
# to an empty window.
|
||||
# A cudagraph padding request: seq_len 0.
|
||||
PADDING_ROWS = 1
|
||||
|
||||
# The Gluon path flattens the KV cache to one page per token.
|
||||
@@ -68,16 +63,10 @@ def _seq_lens() -> list[int]:
|
||||
return [c + QLEN for c in CONTEXT_LENS] + [0] * PADDING_ROWS
|
||||
|
||||
|
||||
def _expected_row_lens() -> list[int]:
|
||||
"""Causal row lengths: row r*QLEN + t sees seq_len_r - (QLEN - 1) + t."""
|
||||
return [max(0, s - (QLEN - 1) + t) for s in _seq_lens() for t in range(QLEN)]
|
||||
|
||||
|
||||
def _run_verify_block():
|
||||
"""Drive the real builder + forward_mqa over one multi-token verify block.
|
||||
|
||||
Returns ``(metadata, captured)`` where ``captured`` holds the ``page_table``,
|
||||
``seq_info`` and ``min_kv_seq_len`` the backend passed to the Gluon kernel.
|
||||
Returns ``(metadata, captured)`` where ``captured`` holds the Gluon kwargs.
|
||||
"""
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
@@ -144,9 +133,10 @@ def _run_verify_block():
|
||||
captured: dict = {}
|
||||
|
||||
def spy(**kwargs):
|
||||
captured.update(kwargs)
|
||||
captured["q_nope"] = kwargs["q_nope"].detach().clone()
|
||||
captured["page_table"] = kwargs["page_table"].detach().clone()
|
||||
captured["seq_info"] = kwargs["seq_info"].detach().clone()
|
||||
captured["min_kv_seq_len"] = kwargs["min_kv_seq_len"]
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
builder = builder_cls(spec, [layer_name], vllm_config, device)
|
||||
@@ -206,12 +196,12 @@ def _run_verify_block():
|
||||
return metadata, captured
|
||||
|
||||
|
||||
def test_verify_flatten_rows_are_causal():
|
||||
"""Verify row t must see the committed prefix plus tokens 0..t, and no more.
|
||||
def test_verify_mtp_uses_native_4d_gluon_entry():
|
||||
"""Verify uses 4-D q with per-request paged_kv metadata.
|
||||
|
||||
Regression guard: before the fix every row of a request got that request's
|
||||
whole paged-KV range, so verify position t attended to the draft tokens
|
||||
after it. Fails on unmodified upstream.
|
||||
Regression guard: before the fix every flattened row of a request got that
|
||||
request's whole paged-KV range, so verify position t attended to the draft
|
||||
tokens after it. The native MTP path delegates causal masking to aiter.
|
||||
"""
|
||||
metadata, captured = _run_verify_block()
|
||||
|
||||
@@ -219,50 +209,37 @@ def test_verify_flatten_rows_are_causal():
|
||||
assert decode is not None, "batch was not classified as a decode"
|
||||
assert decode.max_qo_len == QLEN, (
|
||||
f"expected a {QLEN}-token verify block, got max_qo_len="
|
||||
f"{decode.max_qo_len}; the flatten under test was not reached"
|
||||
f"{decode.max_qo_len}; the MTP path under test was not reached"
|
||||
)
|
||||
assert captured, "forward_mqa did not reach the Gluon kernel"
|
||||
|
||||
seq_lens = _seq_lens()
|
||||
indptr = captured["seq_info"].tolist()
|
||||
num_reqs = len(seq_lens)
|
||||
|
||||
q_nope = captured["q_nope"]
|
||||
assert q_nope.dim() == 4, (
|
||||
f"verify must pass 4-D q to mla_gluon, got {q_nope.dim()}-D shape "
|
||||
f"{tuple(q_nope.shape)}"
|
||||
)
|
||||
assert q_nope.shape[:2] == (num_reqs, QLEN), (
|
||||
f"expected q shape [num_reqs={num_reqs}, qlen={QLEN}, ...], "
|
||||
f"got {tuple(q_nope.shape)}"
|
||||
)
|
||||
assert q_nope.shape[2] == NUM_QUERY_HEADS
|
||||
|
||||
assert captured["use_2d_view"] is False
|
||||
page_table = captured["page_table"]
|
||||
got_row_lens = [indptr[i + 1] - indptr[i] for i in range(len(indptr) - 1)]
|
||||
want_row_lens = _expected_row_lens()
|
||||
|
||||
assert got_row_lens == want_row_lens, (
|
||||
"AITER MLA verify flatten is not causal: verify row r*qlen+t must get "
|
||||
f"seq_len_r - {QLEN - 1} + t KV entries.\n"
|
||||
f" seq_lens {seq_lens}\n"
|
||||
f" got {got_row_lens}\n"
|
||||
f" expected {want_row_lens}\n"
|
||||
"Rows longer than expected let a verify position attend to the draft "
|
||||
"tokens it is supposed to be checking."
|
||||
seq_info = captured["seq_info"]
|
||||
assert page_table.dim() == 1, (
|
||||
f"verify MTP passes the 1-D paged_kv_indices buffer, got {page_table.dim()}-D"
|
||||
)
|
||||
|
||||
# arange_block_indices lays request r's pages out ascending from
|
||||
# r * max_blocks, so each causal window must be that slice's prefix.
|
||||
max_blocks = max(seq_lens)
|
||||
for r, seq_len in enumerate(seq_lens):
|
||||
for t in range(QLEN):
|
||||
row = r * QLEN + t
|
||||
window = page_table[indptr[row] : indptr[row + 1]].tolist()
|
||||
n = max(0, seq_len - (QLEN - 1) + t)
|
||||
want = [r * max_blocks + j for j in range(n)]
|
||||
assert window == want, (
|
||||
f"request {r} (seq_len {seq_len}) verify token {t} reads KV "
|
||||
f"pages {window}, expected the ascending causal prefix {want}"
|
||||
)
|
||||
|
||||
padding_rows = got_row_lens[len(CONTEXT_LENS) * QLEN :]
|
||||
assert padding_rows == [0] * (PADDING_ROWS * QLEN), (
|
||||
f"cudagraph padding requests (seq_len 0) must clamp to empty windows, "
|
||||
f"got {padding_rows}"
|
||||
assert seq_info.dim() == 1 and seq_info.numel() == num_reqs + 1, (
|
||||
f"verify MTP passes paged_kv_indptr [num_reqs + 1], got shape "
|
||||
f"{tuple(seq_info.shape)}"
|
||||
)
|
||||
|
||||
# min_kv_seq_len tells Gluon how short the shortest row is, so it must be
|
||||
# the minimum over the causal rows actually submitted, not over the
|
||||
# per-request lengths they were cut from.
|
||||
assert captured["min_kv_seq_len"] == min(want_row_lens), (
|
||||
f"min_kv_seq_len={captured['min_kv_seq_len']} does not match the "
|
||||
f"shortest submitted row ({min(want_row_lens)})"
|
||||
assert decode is not None and decode.paged_kv_indptr is not None
|
||||
assert torch.equal(seq_info, decode.paged_kv_indptr[: num_reqs + 1])
|
||||
assert torch.equal(
|
||||
page_table[: int(seq_info[-1].item())],
|
||||
decode.paged_kv_indices[: int(seq_info[-1].item())],
|
||||
)
|
||||
|
||||
@@ -40,7 +40,6 @@ class _ExpandPageIndicesKernel:
|
||||
block_table_tensor,
|
||||
stride,
|
||||
paged_kv_indptr,
|
||||
seq_lens_for_kernel,
|
||||
*,
|
||||
KERNEL_BLOCK_SIZE,
|
||||
BLOCK_SIZE,
|
||||
@@ -48,7 +47,9 @@ class _ExpandPageIndicesKernel:
|
||||
self.kernel_block_size = KERNEL_BLOCK_SIZE
|
||||
for req_idx in range(self.grid[0]):
|
||||
out_start = int(paged_kv_indptr[req_idx].item())
|
||||
seq_len = int(seq_lens_for_kernel[req_idx].item())
|
||||
seq_len = int(
|
||||
(paged_kv_indptr[req_idx + 1] - paged_kv_indptr[req_idx]).item()
|
||||
)
|
||||
for token_idx in range(seq_len):
|
||||
block_id = int(
|
||||
block_table_tensor[req_idx, token_idx // KERNEL_BLOCK_SIZE].item()
|
||||
@@ -249,6 +250,51 @@ def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch):
|
||||
assert get_mla_metadata_v1.call_args.kwargs["uni_seqlen_qo"] == 4
|
||||
|
||||
|
||||
def test_min_kv_seq_len_ignores_cudagraph_padding_rows(monkeypatch):
|
||||
"""min_kv_seq_len must not be driven by padded dummy requests.
|
||||
|
||||
Full-CG verify pins zero-qo rows to max_qo_len for paged_kv metadata; taking
|
||||
per_req_len.min() over all rows then reports qlen instead of a real KV length
|
||||
and collapses Gluon KV-split parallelism on single-request decode.
|
||||
"""
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=mock.MagicMock()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
|
||||
)
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_gluon_mla_decode_supported", lambda: True)
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_aiter_mla_small_head_mode", lambda: "auto")
|
||||
|
||||
mtp_qlen = 8
|
||||
num_reqs = 8
|
||||
active_seq_len = 1032
|
||||
seq_lens = torch.tensor([active_seq_len] + [0] * (num_reqs - 1), dtype=torch.int32)
|
||||
query_start_loc = torch.tensor(
|
||||
[0, mtp_qlen] + [mtp_qlen] * (num_reqs - 1), dtype=torch.int32
|
||||
)
|
||||
|
||||
metadata = AiterMLAMetadataBuilder._build_decode(
|
||||
_builder(
|
||||
mtp_decode_qlen=mtp_qlen,
|
||||
has_full_cudagraphs=True,
|
||||
max_decode_rows=num_reqs,
|
||||
num_heads=12,
|
||||
),
|
||||
block_table_tensor=torch.zeros(num_reqs, active_seq_len, dtype=torch.int32),
|
||||
seq_lens_device=seq_lens,
|
||||
max_seq_len=active_seq_len,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
query_start_loc_device=query_start_loc,
|
||||
num_decode_tokens=num_reqs * mtp_qlen,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
assert metadata.min_kv_seq_len == active_seq_len
|
||||
|
||||
|
||||
def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
|
||||
monkeypatch,
|
||||
):
|
||||
|
||||
@@ -729,7 +729,6 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
block_table_tensor,
|
||||
block_table_tensor.stride(0),
|
||||
paged_kv_indptr,
|
||||
seq_lens_for_kernel,
|
||||
KERNEL_BLOCK_SIZE=self.kernel_block_size,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
@@ -834,6 +833,26 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
)
|
||||
has_persistent_metadata = True
|
||||
|
||||
# Small-head multi-token verify uses mla_gluon's 4-D MTP entry over the
|
||||
# ordinary per-request paged-KV view, so there is no expanded per-token
|
||||
# buffer to build here. mla_gluon still wants a lower bound on the KV
|
||||
# length it is asked to split; on the verify path that bound is over
|
||||
# active requests, not cudagraph padding rows pinned to max_qo_len.
|
||||
# The .item() below runs in the builder, outside the captured region,
|
||||
# so it does not abort HIP graph capture the way the per-layer syncs
|
||||
# in forward_mqa did.
|
||||
min_kv_seq_len = 1
|
||||
if AiterMLAHelper.use_gluon_verify(
|
||||
self.num_heads, int(max_qo_len), self._kv_cache_dtype_str
|
||||
):
|
||||
per_req_len = paged_kv_indptr[1:] - paged_kv_indptr[:-1]
|
||||
if pad_uniform_mtp:
|
||||
active = qo_lens_device > 0
|
||||
if active.any():
|
||||
min_kv_seq_len = int(per_req_len[active].min().item())
|
||||
else:
|
||||
min_kv_seq_len = int(per_req_len.min().item())
|
||||
|
||||
attn_metadata = AiterMLADecodeMetadata(
|
||||
block_table=block_table_tensor,
|
||||
seq_lens=seq_lens_for_kernel,
|
||||
@@ -843,6 +862,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
qo_indptr=qo_indptr,
|
||||
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
|
||||
max_qo_len=max_qo_len,
|
||||
min_kv_seq_len=min_kv_seq_len,
|
||||
use_gluon_decode=use_gluon_decode,
|
||||
attn_out_dtype=self.decode_attn_out_dtype,
|
||||
has_persistent_metadata=has_persistent_metadata,
|
||||
@@ -884,7 +904,6 @@ def _expand_page_indices_kernel(
|
||||
block_table,
|
||||
block_table_stride,
|
||||
cu_num_tokens,
|
||||
seq_lens,
|
||||
KERNEL_BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
@@ -904,7 +923,7 @@ def _expand_page_indices_kernel(
|
||||
req_idx = tl.program_id(0)
|
||||
row_ptr = block_table + req_idx * block_table_stride
|
||||
start_idx = tl.load(cu_num_tokens + req_idx)
|
||||
num_tokens = tl.load(seq_lens + req_idx)
|
||||
num_tokens = tl.load(cu_num_tokens + req_idx + 1) - start_idx
|
||||
|
||||
offset = tl.arange(0, BLOCK_SIZE)
|
||||
for i in tl.range(0, num_tokens, BLOCK_SIZE):
|
||||
@@ -1056,11 +1075,12 @@ class AiterMLAHelper:
|
||||
|
||||
@staticmethod
|
||||
def use_gluon_verify(num_heads: int, max_qo_len: int, kv_cache_dtype: str) -> bool:
|
||||
"""Whether a small-head multi-token verify is flattened onto Gluon.
|
||||
"""Whether a small-head multi-token verify uses native Gluon MTP.
|
||||
|
||||
bf16 has no gqa<16, qseqlen>1 asm kernel, so the verify is flattened
|
||||
into per-token Gluon decodes. fp8 has one via the q-row fold and must
|
||||
not come here: the flatten hands Gluon the batch size its fp8 regime
|
||||
bf16 has no gqa<16, qseqlen>1 asm kernel, so verify goes through
|
||||
``mla_gluon``'s 4-D MTP entry (``q`` shaped ``[batch, qlen, nhead, dim]``)
|
||||
with ``use_2d_view=False``. fp8 has one via the q-row fold and must not
|
||||
come here: the MTP path hands Gluon the batch size its fp8 regime
|
||||
asserts against. A predicate rather than inline in forward_mqa so the
|
||||
builder sees the same answer the impl acts on.
|
||||
"""
|
||||
@@ -1391,19 +1411,17 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
return o, None
|
||||
|
||||
# 12-head (<16) multi-token verify (DSpark): the asm path has no
|
||||
# gqa<16, qseqlen>1 kernel. Flatten each verify token to its own
|
||||
# qseqlen=1 gluon decode, mirroring the TRITON_MLA / sparse-backend
|
||||
# flatten but on the fast gluon kernel. The block is causal -- the
|
||||
# target is checking draft tokens, so position t must not see t+1 --
|
||||
# and attention rows are independent, so giving row t the KV range
|
||||
# [0, context + t] is exactly causal multi-token attention.
|
||||
# gqa<16, qseqlen>1 kernel, so the block goes to the gluon kernel's 4-D
|
||||
# MTP entry, which serves a whole (1 + num_spec) block in one launch.
|
||||
# The block is causal -- the target is checking draft tokens, so
|
||||
# position t must not see t+1 -- and the kernel bounds each query
|
||||
# position's scores itself.
|
||||
# Arch, mode and dtype gating all live in use_gluon_verify, so that the
|
||||
# builder -- which has to know whether the asm decode will run -- sees
|
||||
# the same answer as this branch.
|
||||
if AiterMLAHelper.use_gluon_verify(
|
||||
self.num_heads, int(decode.max_qo_len), self.kv_cache_dtype
|
||||
):
|
||||
qlen = int(decode.max_qo_len)
|
||||
if type(q) is tuple:
|
||||
q_nope, q_pe = q
|
||||
else:
|
||||
@@ -1419,56 +1437,39 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
device=q_nope.device,
|
||||
)
|
||||
kv_buffer = kv_c_and_k_pe_cache.reshape(-1, kv_c_and_k_pe_cache.shape[-1])
|
||||
# Expand per-request paged-KV to per-verify-token. Row r*qlen+t is
|
||||
# request r's verify token t, and seq_lens counts the tokens
|
||||
# scheduled in this step, so a request's KV range already spans its
|
||||
# whole verify block and context_r = seq_len_r - qlen. Token t may
|
||||
# attend to [0, context_r + t], i.e. seq_len_r - (qlen - 1) + t
|
||||
# entries. paged_kv_indices lists a request's pages in ascending
|
||||
# position order, so each row's causal window is a prefix of that
|
||||
# request's slice and only the row length changes. Rows clamp to
|
||||
# zero for cudagraph padding requests, whose seq_len is 0. Fully
|
||||
# vectorized (no host loop).
|
||||
old_indptr = decode.paged_kv_indptr
|
||||
per_req_len = old_indptr[1:] - old_indptr[:-1]
|
||||
dev = q_nope.device
|
||||
row_req = torch.arange(per_req_len.shape[0], device=dev).repeat_interleave(
|
||||
qlen
|
||||
assert attn_metadata.causal, (
|
||||
"AITER MLA small-head verify MTP is causal-only"
|
||||
)
|
||||
row_len = (
|
||||
(
|
||||
per_req_len.unsqueeze(1)
|
||||
- (qlen - 1)
|
||||
+ torch.arange(qlen, device=dev, dtype=per_req_len.dtype)
|
||||
# Hand mla_gluon its 4-D MTP entry instead of an expanded
|
||||
# per-verify-token paged-KV view. The flat query layout is already
|
||||
# row-major (r * qlen + t), so unflatten is a free view. mla_gluon
|
||||
# applies the per-position causal bound
|
||||
# score_end = min(split_kv_end, seq_len - qlen + q_pos + 1)
|
||||
# in-kernel, and seq_lens already spans the verify block, so that
|
||||
# bound is context_r + q_pos + 1 -- the same window the expanded
|
||||
# view supplied by truncating each row's page list.
|
||||
qlen = int(decode.max_qo_len)
|
||||
num_reqs = B // qlen
|
||||
if num_reqs * qlen != B:
|
||||
raise ValueError(
|
||||
f"verify block {B} rows is not a multiple of qlen {qlen}"
|
||||
)
|
||||
.clamp_(min=0)
|
||||
.flatten()
|
||||
)
|
||||
new_indptr = torch.cat([old_indptr.new_zeros(1), row_len.cumsum(0)]).to(
|
||||
torch.int32
|
||||
)
|
||||
total = int(new_indptr[-1].item())
|
||||
within = torch.arange(total, device=dev, dtype=torch.int64) - new_indptr[
|
||||
:-1
|
||||
].to(torch.int64).repeat_interleave(row_len)
|
||||
src = (
|
||||
old_indptr[row_req].to(torch.int64).repeat_interleave(row_len) + within
|
||||
)
|
||||
new_indices = decode.paged_kv_indices[src]
|
||||
assert decode.paged_kv_indptr is not None
|
||||
assert decode.paged_kv_indices is not None
|
||||
mla_gluon = _get_mla_gluon()
|
||||
mla_gluon(
|
||||
q_nope=q_nope,
|
||||
q_pe=q_pe,
|
||||
q_nope=q_nope.unflatten(0, (num_reqs, qlen)),
|
||||
q_pe=q_pe.unflatten(0, (num_reqs, qlen)),
|
||||
kv_c=kv_buffer,
|
||||
o=o,
|
||||
page_table=new_indices,
|
||||
seq_info=new_indptr,
|
||||
o=o.unflatten(0, (num_reqs, qlen)),
|
||||
page_table=decode.paged_kv_indices,
|
||||
seq_info=decode.paged_kv_indptr,
|
||||
sm_scale=self.scale,
|
||||
k_pe=None,
|
||||
kv_pe_offset=self.kv_lora_rank,
|
||||
use_2d_view=False,
|
||||
kv_scale=1.0,
|
||||
min_kv_seq_len=int(row_len.min()),
|
||||
min_kv_seq_len=decode.min_kv_seq_len,
|
||||
)
|
||||
return o, None
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import ClassVar
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
@@ -25,6 +26,7 @@ from vllm.v1.attention.backend import (
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
from vllm.v1.worker.workspace import (
|
||||
current_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
@@ -55,6 +57,34 @@ class TritonMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
|
||||
# forward_mqa, so no intra-block causal masking is required.
|
||||
supports_non_causal_multi_token_decode: ClassVar[bool] = True
|
||||
|
||||
@classmethod
|
||||
def get_cudagraph_support(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> AttentionCGSupport:
|
||||
"""Report UNIFORM_BATCH where a non-causal multi-token block is served.
|
||||
|
||||
``_cudagraph_support`` is a class constant, so serving the DSpark
|
||||
draft's (1 + num_spec) block through the decode path reports
|
||||
UNIFORM_SINGLE_TOKEN_DECODE and, because the engine takes the minimum
|
||||
over all attention groups, downgrades the *whole* engine off full
|
||||
cudagraphs. ``forward_mqa`` flattens that block with
|
||||
``repeat_interleave`` on a Python int and performs no device->host
|
||||
sync, so it does satisfy the UNIFORM_BATCH contract.
|
||||
|
||||
``non_causal_multi_token_decode`` is a KV-cache-group property, not a
|
||||
per-layer one: ``MLAAttentionSpec.merge`` ORs it over every layer in
|
||||
the group, so a group holding both a draft and its target reports it
|
||||
for both. That is the same predicate ``__init__`` below already uses to
|
||||
raise ``reorder_batch_threshold``, so the two stay consistent, but it
|
||||
does mean this lifts a causal target sharing the draft's KV cache group
|
||||
as well.
|
||||
"""
|
||||
if getattr(kv_cache_spec, "non_causal_multi_token_decode", False):
|
||||
return AttentionCGSupport.UNIFORM_BATCH
|
||||
return cls._cudagraph_support
|
||||
|
||||
def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
# DCP local sequence lengths are not advanced between draft steps.
|
||||
|
||||
Reference in New Issue
Block a user