[ROCm][MLA] Reach FULL cudagraphs for AITER MLA speculative decoding (#51171)

Signed-off-by: yudigege86 <naqin@amd.com>
Co-authored-by: yudigege86 <naqin@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Matt Qin
2026-08-29 19:49:55 -07:00
committed by GitHub
co-authored by yudigege86 Cursor Andreas Karatzas
parent b5707bf994
commit dbf662c9e8
4 changed files with 180 additions and 126 deletions
@@ -40,7 +40,6 @@ class _ExpandPageIndicesKernel:
block_table_tensor,
stride,
paged_kv_indptr,
seq_lens_for_kernel,
*,
KERNEL_BLOCK_SIZE,
BLOCK_SIZE,
@@ -48,7 +47,9 @@ class _ExpandPageIndicesKernel:
self.kernel_block_size = KERNEL_BLOCK_SIZE
for req_idx in range(self.grid[0]):
out_start = int(paged_kv_indptr[req_idx].item())
seq_len = int(seq_lens_for_kernel[req_idx].item())
seq_len = int(
(paged_kv_indptr[req_idx + 1] - paged_kv_indptr[req_idx]).item()
)
for token_idx in range(seq_len):
block_id = int(
block_table_tensor[req_idx, token_idx // KERNEL_BLOCK_SIZE].item()
@@ -249,6 +250,51 @@ def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch):
assert get_mla_metadata_v1.call_args.kwargs["uni_seqlen_qo"] == 4
def test_min_kv_seq_len_ignores_cudagraph_padding_rows(monkeypatch):
"""min_kv_seq_len must not be driven by padded dummy requests.
Full-CG verify pins zero-qo rows to max_qo_len for paged_kv metadata; taking
per_req_len.min() over all rows then reports qlen instead of a real KV length
and collapses Gluon KV-split parallelism on single-request decode.
"""
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=mock.MagicMock()),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
monkeypatch.setattr(rocm_aiter_mla, "_gluon_mla_decode_supported", lambda: True)
monkeypatch.setattr(rocm_aiter_mla, "_aiter_mla_small_head_mode", lambda: "auto")
mtp_qlen = 8
num_reqs = 8
active_seq_len = 1032
seq_lens = torch.tensor([active_seq_len] + [0] * (num_reqs - 1), dtype=torch.int32)
query_start_loc = torch.tensor(
[0, mtp_qlen] + [mtp_qlen] * (num_reqs - 1), dtype=torch.int32
)
metadata = AiterMLAMetadataBuilder._build_decode(
_builder(
mtp_decode_qlen=mtp_qlen,
has_full_cudagraphs=True,
max_decode_rows=num_reqs,
num_heads=12,
),
block_table_tensor=torch.zeros(num_reqs, active_seq_len, dtype=torch.int32),
seq_lens_device=seq_lens,
max_seq_len=active_seq_len,
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=num_reqs * mtp_qlen,
dcp_tot_seq_lens_device=None,
)
assert metadata.min_kv_seq_len == active_seq_len
def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
monkeypatch,
):