[ROCm][MiniMax-M3][Spec Decode] Support speculative decode with AITER sparse PA (#47984)

Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Hongxia Yang <hongxia.yang@amd.com>
Co-authored-by: Jun Kang Chow <junkangchow@gmail.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
Tan Pin Siang
2026-07-14 04:12:53 +00:00
committed by GitHub
co-authored by vllmellm Hongxia Yang Jun Kang Chow TJian
parent dcf4072da9
commit 95aab66e95
3 changed files with 182 additions and 15 deletions
+140
View File
@@ -1010,6 +1010,146 @@ def _build_decode_inputs(
return q, block_table, seq_lens, topk_idx, num_pages
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="MiniMax-M3 AITER sparse PA is ROCm-only",
)
def test_aiter_sparse_block_table_handles_padded_decode_rows():
"""Zero-length padded decode rows must produce empty sparse attention."""
from vllm.models.minimax_m3.amd.ops.sparse_pa import (
PAGES_PER_SPARSE_BLOCK,
minimax_m3_build_sparse_block_table,
)
topk_idx = torch.tensor(
[[[1, 0, -1], [0, 1, -1]]],
device="cuda",
dtype=torch.int32,
)
block_table = torch.tensor(
[[5, 7], [11, 13]],
device="cuda",
dtype=torch.int32,
)
seq_lens = torch.tensor([129, 0], device="cuda", dtype=torch.int32)
sparse_bt, sparse_ctx = minimax_m3_build_sparse_block_table(
topk_idx, block_table, seq_lens
)
torch.accelerator.synchronize()
expected_bt = torch.zeros_like(sparse_bt)
expected_ctx = torch.tensor([129, 0], device="cuda", dtype=torch.int32)
for slot, block_id in enumerate((0, 1)):
base_phys = int(block_table[0, block_id]) * PAGES_PER_SPARSE_BLOCK
start = slot * PAGES_PER_SPARSE_BLOCK
expected_bt[0, start : start + PAGES_PER_SPARSE_BLOCK] = torch.arange(
base_phys,
base_phys + PAGES_PER_SPARSE_BLOCK,
device="cuda",
dtype=torch.int32,
)
assert torch.equal(sparse_ctx, expected_ctx)
assert torch.equal(sparse_bt, expected_bt)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="MiniMax-M3 AITER sparse PA is ROCm-only",
)
def test_aiter_decode_sparse_block_table_supports_spec_decode():
"""AITER sparse PA needs one compact page-16 table per speculative query.
The decode indexer flattens speculative rows as
``req_id * decode_query_len + local_q``. This verifies that the ROCm AITER
block-table adapter uses the same mapping before handing rows to Gluon.
"""
from vllm.models.minimax_m3.amd.ops.sparse_pa import (
PAGES_PER_SPARSE_BLOCK,
minimax_m3_build_sparse_block_table_decode,
)
decode_query_len = 4
seq_lens = torch.tensor([260, 132, 0], device="cuda", dtype=torch.int32)
block_table = torch.tensor(
[
[5, 7, 11],
[13, 17, 19],
[0, 0, 0],
],
device="cuda",
dtype=torch.int32,
)
topk_idx = torch.tensor(
[
[
[2, 0, 3, -1],
[0, 2, 3, -1],
[2, 1, 0, -1],
[1, 2, 0, -1],
[1, 0, 2, -1],
[0, 1, 2, -1],
[1, -1, 0, -1],
[0, 2, 1, -1],
[0, 1, -1, -1],
[0, 1, -1, -1],
[0, 1, -1, -1],
[0, 1, -1, -1],
]
],
device="cuda",
dtype=torch.int32,
)
sparse_bt, sparse_ctx = minimax_m3_build_sparse_block_table_decode(
topk_idx, block_table, seq_lens, decode_query_len
)
torch.accelerator.synchronize()
expected_bt = torch.zeros_like(sparse_bt)
expected_ctx = torch.empty_like(sparse_ctx)
topk_cpu = topk_idx.cpu()[0].tolist()
block_table_cpu = block_table.cpu()
seq_lens_cpu = seq_lens.cpu().tolist()
for row, selected_blocks in enumerate(topk_cpu):
req_id = row // decode_query_len
local_q = row - req_id * decode_query_len
query_pos = seq_lens_cpu[req_id] - decode_query_len + local_q
causal_len = max(query_pos + 1, 0)
if causal_len == 0:
expected_ctx[row] = 0
continue
self_block = query_pos // BLOCK_SIZE
valid = [blk for blk in selected_blocks if 0 <= blk <= self_block]
full_blocks = [blk for blk in valid if blk < self_block]
tail_blocks = [blk for blk in valid if blk == self_block]
ordered_blocks = full_blocks + tail_blocks
for slot, block_id in enumerate(ordered_blocks):
base_phys = int(block_table_cpu[req_id, block_id]) * PAGES_PER_SPARSE_BLOCK
start = slot * PAGES_PER_SPARSE_BLOCK
expected_bt[row, start : start + PAGES_PER_SPARSE_BLOCK] = torch.arange(
base_phys,
base_phys + PAGES_PER_SPARSE_BLOCK,
device="cuda",
dtype=torch.int32,
)
if tail_blocks:
expected_ctx[row] = (
len(full_blocks) * BLOCK_SIZE + causal_len - self_block * BLOCK_SIZE
)
else:
expected_ctx[row] = min(len(valid) * BLOCK_SIZE, causal_len)
assert torch.equal(sparse_ctx, expected_ctx)
assert torch.equal(sparse_bt, expected_bt)
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
@pytest.mark.parametrize(
"seq_lens_list",
+41 -10
View File
@@ -49,7 +49,8 @@ def _build_sparse_block_table_kernel(
):
pid_b = tl.program_id(0)
seq_len = tl.load(seq_lens_ptr + pid_b)
last_blk = (seq_len - 1) // SPARSE_BLOCK_SIZE_C
has_tokens = seq_len > 0
last_blk = tl.maximum((seq_len - 1) // SPARSE_BLOCK_SIZE_C, 0)
topk_row = topk_ptr + pid_b * stride_topk_n
bt_row = block_table_ptr + pid_b * stride_bt_b
@@ -57,7 +58,7 @@ def _build_sparse_block_table_kernel(
off_t = tl.arange(0, BLOCK_SIZE_T)
blk = tl.load(topk_row + off_t * stride_topk_k, mask=off_t < max_topk, other=-1)
valid = blk >= 0
valid = has_tokens & (blk >= 0)
is_tail = valid & (blk == last_blk)
is_full = valid & (blk != last_blk)
@@ -82,7 +83,7 @@ def _build_sparse_block_table_kernel(
mask=(off_w >= n_used) & (off_w < row_width),
)
tail_tokens = seq_len - last_blk * SPARSE_BLOCK_SIZE_C
tail_tokens = tl.where(has_tokens, seq_len - last_blk * SPARSE_BLOCK_SIZE_C, 0)
has_tail = tl.sum(is_tail.to(tl.int32), axis=0) > 0
ctx = n_full * SPARSE_BLOCK_SIZE_C + tl.where(has_tail, tail_tokens, 0)
ctx = tl.where(has_tail, ctx, tl.minimum(n_valid * SPARSE_BLOCK_SIZE_C, seq_len))
@@ -140,7 +141,8 @@ def _build_sparse_block_table_prefill_kernel(
pid_n = tl.program_id(0)
req_id = tl.load(req_id_ptr + pid_n)
abs_pos = tl.load(abs_pos_ptr + pid_n)
causal_len = abs_pos + 1
# Padded speculative rows can have negative positions; clamp to empty range.
causal_len = tl.maximum(abs_pos + 1, 0)
self_blk = abs_pos // SPARSE_BLOCK_SIZE_C
topk_row = topk_ptr + pid_n * stride_topk_n
@@ -149,7 +151,7 @@ def _build_sparse_block_table_prefill_kernel(
off_t = tl.arange(0, BLOCK_SIZE_T)
blk = tl.load(topk_row + off_t * stride_topk_k, mask=off_t < max_topk, other=-1)
valid = (blk >= 0) & (blk <= self_blk)
valid = (causal_len > 0) & (blk >= 0) & (blk <= self_blk)
is_tail = valid & (blk == self_blk)
is_full = valid & (blk < self_blk)
@@ -214,6 +216,34 @@ def minimax_m3_build_sparse_block_table_prefill(
return sparse_bt, sparse_ctx
@torch.no_grad()
def minimax_m3_build_sparse_block_table_decode(
topk_idx: torch.Tensor, # [1, batch * decode_query_len, topk]
block_table: torch.Tensor, # [batch, max_blocks]
seq_lens: torch.Tensor, # [batch]
decode_query_len: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Build one page-16 sparse block table row per decode query token."""
if decode_query_len == 1:
return minimax_m3_build_sparse_block_table(topk_idx, block_table, seq_lens)
total_q = topk_idx.shape[1]
expected_q = block_table.shape[0] * decode_query_len
assert total_q == expected_q, (
"MiniMax-M3 decode top-k rows must equal batch * decode_query_len: "
f"{total_q} != {expected_q}"
)
pos = torch.arange(total_q, dtype=torch.int32, device=topk_idx.device)
query_req_id = torch.div(pos, decode_query_len, rounding_mode="floor")
q_offset = pos - query_req_id * decode_query_len
query_abs_pos = (seq_lens[query_req_id] - decode_query_len + q_offset).to(
torch.int32
)
return minimax_m3_build_sparse_block_table_prefill(
topk_idx, block_table, query_req_id, query_abs_pos
)
@triton.jit
def _insert_index_cache_kernel(
index_k_ptr, # [num_tokens, head_dim]
@@ -317,20 +347,21 @@ def _gluon_scale_arg(
@torch.no_grad()
def minimax_m3_sparse_attn_decode_aiter(
q: torch.Tensor, # [batch, num_heads, head_dim]
q: torch.Tensor, # [batch * decode_query_len, num_heads, head_dim]
k_cache: torch.Tensor, # [phys16, num_kv_heads, head_dim // x, 16, x]
v_cache: torch.Tensor, # [phys16, num_kv_heads, 16 // x, head_dim, x]
topk_idx: torch.Tensor, # [1, batch, topk]
topk_idx: torch.Tensor, # [1, batch * decode_query_len, topk]
block_table: torch.Tensor, # [batch, max_blocks]
seq_lens: torch.Tensor, # [batch]
num_kv_heads: int,
sm_scale: float,
output: torch.Tensor, # [batch, num_heads, head_dim]
output: torch.Tensor, # [batch * decode_query_len, num_heads, head_dim]
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
decode_query_len: int = 1,
) -> None:
sparse_bt, sparse_ctx = minimax_m3_build_sparse_block_table(
topk_idx, block_table, seq_lens
sparse_bt, sparse_ctx = minimax_m3_build_sparse_block_table_decode(
topk_idx, block_table, seq_lens, decode_query_len
)
_run_gluon_decode(
q,
@@ -55,11 +55,6 @@ class MiniMaxM3SparseAiterPAImpl(MiniMaxM3SparseImpl):
if main_md.num_decodes > 0:
d = main_md.decode
assert d is not None
if d.decode_query_len != 1:
raise NotImplementedError(
"MiniMax-M3 AITER sparse PA does not support speculative "
f"decode_query_len={d.decode_query_len}"
)
minimax_m3_sparse_attn_decode_aiter(
q[:nd],
k_cache,
@@ -72,6 +67,7 @@ class MiniMaxM3SparseAiterPAImpl(MiniMaxM3SparseImpl):
out[:nd],
k_scale=k_scale,
v_scale=v_scale,
decode_query_len=d.decode_query_len,
)
if main_md.num_prefills > 0: