[ROCm][Perf] Optimize MiniMax-M3 decode indexer and top-k (#54682)

Signed-off-by: fai <fangzhouai@gmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Fangzhou Ai
2026-09-03 13:20:18 -07:00
committed by GitHub
co-authored by Andreas Karatzas OpenAI Codex
parent 2a336d8239
commit cee0f92c02
6 changed files with 2028 additions and 345 deletions
+46 -1
View File
@@ -92,6 +92,13 @@ from vllm.models.minimax_m3.amd.ops import (
gemma_rmsnorm,
swiglu_oai_split,
)
from vllm.models.minimax_m3.amd.ops.sparse_pa import (
minimax_m3_alloc_sparse_block_table,
minimax_m3_sparse_block_page_stride,
)
from vllm.models.minimax_m3.amd.sparse_attention_msa import (
MiniMaxM3SparseAiterPAImpl,
)
from vllm.models.minimax_m3.common.indexer import MiniMaxM3Indexer
from vllm.models.minimax_m3.common.mm_preprocess import (
MiniMaxM3VLDummyInputsBuilder,
@@ -701,6 +708,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
self.kv_cache_k = torch.tensor([])
self.kv_cache_v = torch.tensor([])
self._aiter_sparse_pa_cache_data_ptr = 0
self._aiter_sparse_pa_block_page_stride = 0
# Self-contained nn.Module: owns its side cache, selects its impl in init
# (Triton on ROCm, where the SM100 gate is always False).
self.indexer = MiniMaxM3Indexer(
@@ -796,6 +804,9 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
self.head_dim,
x,
)[v_page_offset:]
self._aiter_sparse_pa_block_page_stride = minimax_m3_sparse_block_page_stride(
self.kv_cache_k, self.kv_cache_v
)
self._aiter_sparse_pa_cache_data_ptr = self.kv_cache.data_ptr()
def get_aiter_sparse_pa_kv_cache(self) -> tuple[torch.Tensor, torch.Tensor]:
@@ -1020,9 +1031,43 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
# top-k into the shared ``topk_indices_buffer``; the attend reads it back.
# When skip_index_topk is set (ATOM index_topk_freq), reuse the selection
# the preceding compute layer wrote into the shared buffer this forward.
decode_sparse_table: tuple[torch.Tensor, torch.Tensor] | None = None
if not self.skip_index_topk:
assert index_query is not None
self.indexer(index_query)
attn_metadata = get_forward_context().attn_metadata
if self.use_aiter_sparse_pa and isinstance(attn_metadata, dict):
main_md = attn_metadata[self.layer_name]
assert isinstance(main_md, MiniMaxM3SparseMetadata)
if main_md.num_decodes > 0:
d = main_md.decode
assert d is not None
topk = self.topk_indices_buffer
assert topk is not None
decode_sparse_table = minimax_m3_alloc_sparse_block_table(
topk[:, : main_md.num_decode_tokens, :]
)
block_page_stride = self._aiter_sparse_pa_block_page_stride
assert block_page_stride > 0
self.indexer(
index_query,
attention_block_table=d.block_table,
sparse_block_table_out=decode_sparse_table[0],
sparse_context_lens_out=decode_sparse_table[1],
block_page_stride=block_page_stride,
)
else:
self.indexer(index_query)
else:
self.indexer(index_query)
if self.use_aiter_sparse_pa:
assert isinstance(self.impl, MiniMaxM3SparseAiterPAImpl)
return self.impl.forward(
self,
query,
self.kv_cache,
output,
decode_sparse_table=decode_sparse_table,
)
return self.impl.forward(self, query, self.kv_cache, output)
File diff suppressed because it is too large Load Diff
+60 -19
View File
@@ -32,14 +32,13 @@ def _is_fp8_kv_cache_tensor(kv_cache: torch.Tensor) -> bool:
@triton.jit
def _write_sparse_block_table_row(
topk_row, # [topk] int32, selected logical block ids for this query
def _write_sparse_block_table_row_from_values(
blk, # [BLOCK_SIZE_T] int32 selected logical block ids for this query
bt_row, # [max_blocks] int32, this request's logical page table
sbt_row, # [topk * PAGES_PER_BLOCK] int32, physical 16-page table
ctx_ptr, # int32, this query's attended context length
abs_pos, # absolute position of this query token, may be negative
max_topk,
stride_topk_k,
SPARSE_BLOCK_SIZE_C: tl.constexpr,
PAGES_PER_BLOCK: tl.constexpr,
BLOCK_PAGE_STRIDE: tl.constexpr,
@@ -56,8 +55,7 @@ def _write_sparse_block_table_row(
self_blk = abs_pos // SPARSE_BLOCK_SIZE_C
off_t = tl.arange(0, BLOCK_SIZE_T)
blk = tl.load(topk_row + off_t * stride_topk_k, mask=off_t < max_topk, other=-1)
valid = (causal_len > 0) & (blk >= 0) & (blk <= self_blk)
valid = (off_t < max_topk) & (causal_len > 0) & (blk >= 0) & (blk <= self_blk)
is_tail = valid & (blk == self_blk)
is_full = valid & (blk < self_blk)
@@ -89,6 +87,36 @@ def _write_sparse_block_table_row(
tl.store(ctx_ptr, ctx)
@triton.jit
def _write_sparse_block_table_row(
topk_row, # [topk] int32, selected logical block ids for this query
bt_row, # [max_blocks] int32, this request's logical page table
sbt_row, # [topk * PAGES_PER_BLOCK] int32, physical 16-page table
ctx_ptr, # int32, this query's attended context length
abs_pos, # absolute position of this query token, may be negative
max_topk,
stride_topk_k,
SPARSE_BLOCK_SIZE_C: tl.constexpr,
PAGES_PER_BLOCK: tl.constexpr,
BLOCK_PAGE_STRIDE: tl.constexpr,
BLOCK_SIZE_T: tl.constexpr,
):
off_t = tl.arange(0, BLOCK_SIZE_T)
blk = tl.load(topk_row + off_t * stride_topk_k, mask=off_t < max_topk, other=-1)
_write_sparse_block_table_row_from_values(
blk,
bt_row,
sbt_row,
ctx_ptr,
abs_pos,
max_topk,
SPARSE_BLOCK_SIZE_C,
PAGES_PER_BLOCK,
BLOCK_PAGE_STRIDE,
BLOCK_SIZE_T,
)
@triton.jit
def _build_sparse_block_table_kernel(
topk_ptr, # [1, batch * DECODE_QUERY_LEN, topk] int32, logical block ids
@@ -169,7 +197,7 @@ def _build_sparse_block_table_prefill_kernel(
)
def _alloc_sparse_block_table(
def minimax_m3_alloc_sparse_block_table(
topk_idx: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
assert topk_idx.shape[0] == 1, "AITER sparse PA requires num_kv_heads == 1"
@@ -190,7 +218,7 @@ def minimax_m3_build_sparse_block_table_prefill(
block_page_stride: int = PAGES_PER_SPARSE_BLOCK,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Build one page-16 sparse block table row per prefill query token."""
sparse_bt, sparse_ctx = _alloc_sparse_block_table(topk_idx)
sparse_bt, sparse_ctx = minimax_m3_alloc_sparse_block_table(topk_idx)
topk = topk_idx.shape[-1]
_build_sparse_block_table_prefill_kernel[(topk_idx.shape[1],)](
topk_idx,
@@ -227,7 +255,7 @@ def minimax_m3_build_sparse_block_table_decode(
"MiniMax-M3 decode top-k rows must equal batch * decode_query_len: "
f"{total_q} != {expected_q}"
)
sparse_bt, sparse_ctx = _alloc_sparse_block_table(topk_idx)
sparse_bt, sparse_ctx = minimax_m3_alloc_sparse_block_table(topk_idx)
topk = topk_idx.shape[-1]
_build_sparse_block_table_kernel[(total_q,)](
topk_idx,
@@ -328,7 +356,9 @@ def _sides_are_packed(k_cache: torch.Tensor, v_cache: torch.Tensor) -> bool:
return v_cache.shape[0] != k_cache.shape[0]
def _block_page_stride(k_cache: torch.Tensor, v_cache: torch.Tensor) -> int:
def minimax_m3_sparse_block_page_stride(
k_cache: torch.Tensor, v_cache: torch.Tensor
) -> int:
"""How many page ids one sparse block spans.
Each side owns ``PAGES_PER_SPARSE_BLOCK`` pages. When the sides are dense
@@ -384,20 +414,31 @@ def minimax_m3_sparse_attn_decode_aiter(
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
decode_query_len: int = 1,
sparse_block_table: torch.Tensor | None = None,
sparse_context_lens: torch.Tensor | None = None,
) -> None:
sparse_bt, sparse_ctx = minimax_m3_build_sparse_block_table_decode(
topk_idx,
block_table,
seq_lens,
decode_query_len,
_block_page_stride(k_cache, v_cache),
)
if (sparse_block_table is None) != (sparse_context_lens is None):
raise ValueError(
"MiniMax-M3 prepared sparse block table and context lengths "
"must be provided together"
)
if sparse_block_table is None:
sparse_block_table, sparse_context_lens = (
minimax_m3_build_sparse_block_table_decode(
topk_idx,
block_table,
seq_lens,
decode_query_len,
minimax_m3_sparse_block_page_stride(k_cache, v_cache),
)
)
assert sparse_context_lens is not None
_run_gluon_decode(
q,
k_cache,
v_cache,
sparse_bt,
sparse_ctx,
sparse_block_table,
sparse_context_lens,
num_kv_heads,
sm_scale,
output,
@@ -426,7 +467,7 @@ def minimax_m3_sparse_attn_prefill_aiter(
block_table,
query_req_id,
query_abs_pos,
_block_page_stride(k_cache, v_cache),
minimax_m3_sparse_block_page_stride(k_cache, v_cache),
)
_run_gluon_decode(
q,
@@ -25,6 +25,7 @@ class MiniMaxM3SparseAiterPAImpl(MiniMaxM3SparseImpl):
output: torch.Tensor,
*,
query_fp8: torch.Tensor | None = None,
decode_sparse_table: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> torch.Tensor:
from vllm.models.minimax_m3.amd.ops.sparse_pa import (
minimax_m3_sparse_attn_decode_aiter,
@@ -70,6 +71,12 @@ class MiniMaxM3SparseAiterPAImpl(MiniMaxM3SparseImpl):
k_scale=k_scale,
v_scale=v_scale,
decode_query_len=d.decode_query_len,
sparse_block_table=(
decode_sparse_table[0] if decode_sparse_table is not None else None
),
sparse_context_lens=(
decode_sparse_table[1] if decode_sparse_table is not None else None
),
)
if main_md.num_prefills > 0:
+71 -1
View File
@@ -369,6 +369,18 @@ class MiniMaxM3IndexerImpl(nn.Module):
# Shared, stable-address top-k output buffer (set by the model for the
# cudagraph-safe MSA impl); None -> impl allocates fresh (eager).
self.topk_indices_buffer = topk_indices_buffer
topk_completion_counter = None
if current_platform.is_rocm() and topk_indices_buffer is not None:
topk_completion_counter = torch.zeros(
topk_indices_buffer.shape[:2],
dtype=torch.int32,
device=topk_indices_buffer.device,
)
self.register_buffer(
"topk_completion_counter",
topk_completion_counter,
persistent=False,
)
# Owns the side cache (registers itself in the static forward context).
self.index_cache = MiniMaxM3IndexerCache(
head_dim=index_head_dim,
@@ -392,6 +404,11 @@ class MiniMaxM3IndexerTritonImpl(MiniMaxM3IndexerImpl):
def forward(
self,
index_query: torch.Tensor,
*,
attention_block_table: torch.Tensor | None = None,
sparse_block_table_out: torch.Tensor | None = None,
sparse_context_lens_out: torch.Tensor | None = None,
block_page_stride: int | None = None,
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
@@ -417,6 +434,19 @@ class MiniMaxM3IndexerTritonImpl(MiniMaxM3IndexerImpl):
if index_md.num_decodes > 0:
d = index_md.decode
assert d is not None
fused_sparse_kwargs = {}
if attention_block_table is not None:
fused_sparse_kwargs = {
"attention_block_table": attention_block_table,
"sparse_block_table_out": sparse_block_table_out,
"sparse_context_lens_out": sparse_context_lens_out,
"block_page_stride": block_page_stride,
}
decode_backend_kwargs = {}
if current_platform.is_rocm():
decode_backend_kwargs["completion_counter"] = (
self.topk_completion_counter
)
decode_topk = minimax_m3_index_decode(
iq[:nd],
kv,
@@ -430,6 +460,8 @@ class MiniMaxM3IndexerTritonImpl(MiniMaxM3IndexerImpl):
d.decode_query_len,
d.max_decode_query_len,
out=buf_htk,
**fused_sparse_kwargs,
**decode_backend_kwargs,
)
if index_md.num_prefills > 0:
p = index_md.prefill
@@ -567,5 +599,43 @@ class MiniMaxM3Indexer(nn.Module):
def forward(
self,
index_query: torch.Tensor,
*,
attention_block_table: torch.Tensor | None = None,
sparse_block_table_out: torch.Tensor | None = None,
sparse_context_lens_out: torch.Tensor | None = None,
block_page_stride: int | None = None,
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
return self.impl(index_query)
if attention_block_table is None and all(
arg is None
for arg in (
sparse_block_table_out,
sparse_context_lens_out,
block_page_stride,
)
):
return self.impl(index_query)
if attention_block_table is None or any(
arg is None
for arg in (
sparse_block_table_out,
sparse_context_lens_out,
block_page_stride,
)
):
raise ValueError(
"MiniMax-M3 fused decode sparse-table arguments must be "
"provided together"
)
if not current_platform.is_rocm() or not isinstance(
self.impl, MiniMaxM3IndexerTritonImpl
):
raise ValueError(
"MiniMax-M3 fused decode sparse-table construction is ROCm-only"
)
return self.impl(
index_query,
attention_block_table=attention_block_table,
sparse_block_table_out=sparse_block_table_out,
sparse_context_lens_out=sparse_context_lens_out,
block_page_stride=block_page_stride,
)