[ROCm][Perf] Optimize MiniMax-M3 decode indexer and top-k (#54682)
Signed-off-by: fai <fangzhouai@gmail.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Andreas Karatzas
OpenAI Codex
parent
2a336d8239
commit
cee0f92c02
@@ -92,6 +92,13 @@ from vllm.models.minimax_m3.amd.ops import (
|
||||
gemma_rmsnorm,
|
||||
swiglu_oai_split,
|
||||
)
|
||||
from vllm.models.minimax_m3.amd.ops.sparse_pa import (
|
||||
minimax_m3_alloc_sparse_block_table,
|
||||
minimax_m3_sparse_block_page_stride,
|
||||
)
|
||||
from vllm.models.minimax_m3.amd.sparse_attention_msa import (
|
||||
MiniMaxM3SparseAiterPAImpl,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.indexer import MiniMaxM3Indexer
|
||||
from vllm.models.minimax_m3.common.mm_preprocess import (
|
||||
MiniMaxM3VLDummyInputsBuilder,
|
||||
@@ -701,6 +708,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
self.kv_cache_k = torch.tensor([])
|
||||
self.kv_cache_v = torch.tensor([])
|
||||
self._aiter_sparse_pa_cache_data_ptr = 0
|
||||
self._aiter_sparse_pa_block_page_stride = 0
|
||||
# Self-contained nn.Module: owns its side cache, selects its impl in init
|
||||
# (Triton on ROCm, where the SM100 gate is always False).
|
||||
self.indexer = MiniMaxM3Indexer(
|
||||
@@ -796,6 +804,9 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
self.head_dim,
|
||||
x,
|
||||
)[v_page_offset:]
|
||||
self._aiter_sparse_pa_block_page_stride = minimax_m3_sparse_block_page_stride(
|
||||
self.kv_cache_k, self.kv_cache_v
|
||||
)
|
||||
self._aiter_sparse_pa_cache_data_ptr = self.kv_cache.data_ptr()
|
||||
|
||||
def get_aiter_sparse_pa_kv_cache(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -1020,9 +1031,43 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
# top-k into the shared ``topk_indices_buffer``; the attend reads it back.
|
||||
# When skip_index_topk is set (ATOM index_topk_freq), reuse the selection
|
||||
# the preceding compute layer wrote into the shared buffer this forward.
|
||||
decode_sparse_table: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
if not self.skip_index_topk:
|
||||
assert index_query is not None
|
||||
self.indexer(index_query)
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if self.use_aiter_sparse_pa and isinstance(attn_metadata, dict):
|
||||
main_md = attn_metadata[self.layer_name]
|
||||
assert isinstance(main_md, MiniMaxM3SparseMetadata)
|
||||
if main_md.num_decodes > 0:
|
||||
d = main_md.decode
|
||||
assert d is not None
|
||||
topk = self.topk_indices_buffer
|
||||
assert topk is not None
|
||||
decode_sparse_table = minimax_m3_alloc_sparse_block_table(
|
||||
topk[:, : main_md.num_decode_tokens, :]
|
||||
)
|
||||
block_page_stride = self._aiter_sparse_pa_block_page_stride
|
||||
assert block_page_stride > 0
|
||||
self.indexer(
|
||||
index_query,
|
||||
attention_block_table=d.block_table,
|
||||
sparse_block_table_out=decode_sparse_table[0],
|
||||
sparse_context_lens_out=decode_sparse_table[1],
|
||||
block_page_stride=block_page_stride,
|
||||
)
|
||||
else:
|
||||
self.indexer(index_query)
|
||||
else:
|
||||
self.indexer(index_query)
|
||||
if self.use_aiter_sparse_pa:
|
||||
assert isinstance(self.impl, MiniMaxM3SparseAiterPAImpl)
|
||||
return self.impl.forward(
|
||||
self,
|
||||
query,
|
||||
self.kv_cache,
|
||||
output,
|
||||
decode_sparse_table=decode_sparse_table,
|
||||
)
|
||||
return self.impl.forward(self, query, self.kv_cache, output)
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -32,14 +32,13 @@ def _is_fp8_kv_cache_tensor(kv_cache: torch.Tensor) -> bool:
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _write_sparse_block_table_row(
|
||||
topk_row, # [topk] int32, selected logical block ids for this query
|
||||
def _write_sparse_block_table_row_from_values(
|
||||
blk, # [BLOCK_SIZE_T] int32 selected logical block ids for this query
|
||||
bt_row, # [max_blocks] int32, this request's logical page table
|
||||
sbt_row, # [topk * PAGES_PER_BLOCK] int32, physical 16-page table
|
||||
ctx_ptr, # int32, this query's attended context length
|
||||
abs_pos, # absolute position of this query token, may be negative
|
||||
max_topk,
|
||||
stride_topk_k,
|
||||
SPARSE_BLOCK_SIZE_C: tl.constexpr,
|
||||
PAGES_PER_BLOCK: tl.constexpr,
|
||||
BLOCK_PAGE_STRIDE: tl.constexpr,
|
||||
@@ -56,8 +55,7 @@ def _write_sparse_block_table_row(
|
||||
self_blk = abs_pos // SPARSE_BLOCK_SIZE_C
|
||||
|
||||
off_t = tl.arange(0, BLOCK_SIZE_T)
|
||||
blk = tl.load(topk_row + off_t * stride_topk_k, mask=off_t < max_topk, other=-1)
|
||||
valid = (causal_len > 0) & (blk >= 0) & (blk <= self_blk)
|
||||
valid = (off_t < max_topk) & (causal_len > 0) & (blk >= 0) & (blk <= self_blk)
|
||||
is_tail = valid & (blk == self_blk)
|
||||
is_full = valid & (blk < self_blk)
|
||||
|
||||
@@ -89,6 +87,36 @@ def _write_sparse_block_table_row(
|
||||
tl.store(ctx_ptr, ctx)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _write_sparse_block_table_row(
|
||||
topk_row, # [topk] int32, selected logical block ids for this query
|
||||
bt_row, # [max_blocks] int32, this request's logical page table
|
||||
sbt_row, # [topk * PAGES_PER_BLOCK] int32, physical 16-page table
|
||||
ctx_ptr, # int32, this query's attended context length
|
||||
abs_pos, # absolute position of this query token, may be negative
|
||||
max_topk,
|
||||
stride_topk_k,
|
||||
SPARSE_BLOCK_SIZE_C: tl.constexpr,
|
||||
PAGES_PER_BLOCK: tl.constexpr,
|
||||
BLOCK_PAGE_STRIDE: tl.constexpr,
|
||||
BLOCK_SIZE_T: tl.constexpr,
|
||||
):
|
||||
off_t = tl.arange(0, BLOCK_SIZE_T)
|
||||
blk = tl.load(topk_row + off_t * stride_topk_k, mask=off_t < max_topk, other=-1)
|
||||
_write_sparse_block_table_row_from_values(
|
||||
blk,
|
||||
bt_row,
|
||||
sbt_row,
|
||||
ctx_ptr,
|
||||
abs_pos,
|
||||
max_topk,
|
||||
SPARSE_BLOCK_SIZE_C,
|
||||
PAGES_PER_BLOCK,
|
||||
BLOCK_PAGE_STRIDE,
|
||||
BLOCK_SIZE_T,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _build_sparse_block_table_kernel(
|
||||
topk_ptr, # [1, batch * DECODE_QUERY_LEN, topk] int32, logical block ids
|
||||
@@ -169,7 +197,7 @@ def _build_sparse_block_table_prefill_kernel(
|
||||
)
|
||||
|
||||
|
||||
def _alloc_sparse_block_table(
|
||||
def minimax_m3_alloc_sparse_block_table(
|
||||
topk_idx: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert topk_idx.shape[0] == 1, "AITER sparse PA requires num_kv_heads == 1"
|
||||
@@ -190,7 +218,7 @@ def minimax_m3_build_sparse_block_table_prefill(
|
||||
block_page_stride: int = PAGES_PER_SPARSE_BLOCK,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Build one page-16 sparse block table row per prefill query token."""
|
||||
sparse_bt, sparse_ctx = _alloc_sparse_block_table(topk_idx)
|
||||
sparse_bt, sparse_ctx = minimax_m3_alloc_sparse_block_table(topk_idx)
|
||||
topk = topk_idx.shape[-1]
|
||||
_build_sparse_block_table_prefill_kernel[(topk_idx.shape[1],)](
|
||||
topk_idx,
|
||||
@@ -227,7 +255,7 @@ def minimax_m3_build_sparse_block_table_decode(
|
||||
"MiniMax-M3 decode top-k rows must equal batch * decode_query_len: "
|
||||
f"{total_q} != {expected_q}"
|
||||
)
|
||||
sparse_bt, sparse_ctx = _alloc_sparse_block_table(topk_idx)
|
||||
sparse_bt, sparse_ctx = minimax_m3_alloc_sparse_block_table(topk_idx)
|
||||
topk = topk_idx.shape[-1]
|
||||
_build_sparse_block_table_kernel[(total_q,)](
|
||||
topk_idx,
|
||||
@@ -328,7 +356,9 @@ def _sides_are_packed(k_cache: torch.Tensor, v_cache: torch.Tensor) -> bool:
|
||||
return v_cache.shape[0] != k_cache.shape[0]
|
||||
|
||||
|
||||
def _block_page_stride(k_cache: torch.Tensor, v_cache: torch.Tensor) -> int:
|
||||
def minimax_m3_sparse_block_page_stride(
|
||||
k_cache: torch.Tensor, v_cache: torch.Tensor
|
||||
) -> int:
|
||||
"""How many page ids one sparse block spans.
|
||||
|
||||
Each side owns ``PAGES_PER_SPARSE_BLOCK`` pages. When the sides are dense
|
||||
@@ -384,20 +414,31 @@ def minimax_m3_sparse_attn_decode_aiter(
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
decode_query_len: int = 1,
|
||||
sparse_block_table: torch.Tensor | None = None,
|
||||
sparse_context_lens: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
sparse_bt, sparse_ctx = minimax_m3_build_sparse_block_table_decode(
|
||||
topk_idx,
|
||||
block_table,
|
||||
seq_lens,
|
||||
decode_query_len,
|
||||
_block_page_stride(k_cache, v_cache),
|
||||
)
|
||||
if (sparse_block_table is None) != (sparse_context_lens is None):
|
||||
raise ValueError(
|
||||
"MiniMax-M3 prepared sparse block table and context lengths "
|
||||
"must be provided together"
|
||||
)
|
||||
if sparse_block_table is None:
|
||||
sparse_block_table, sparse_context_lens = (
|
||||
minimax_m3_build_sparse_block_table_decode(
|
||||
topk_idx,
|
||||
block_table,
|
||||
seq_lens,
|
||||
decode_query_len,
|
||||
minimax_m3_sparse_block_page_stride(k_cache, v_cache),
|
||||
)
|
||||
)
|
||||
assert sparse_context_lens is not None
|
||||
_run_gluon_decode(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
sparse_bt,
|
||||
sparse_ctx,
|
||||
sparse_block_table,
|
||||
sparse_context_lens,
|
||||
num_kv_heads,
|
||||
sm_scale,
|
||||
output,
|
||||
@@ -426,7 +467,7 @@ def minimax_m3_sparse_attn_prefill_aiter(
|
||||
block_table,
|
||||
query_req_id,
|
||||
query_abs_pos,
|
||||
_block_page_stride(k_cache, v_cache),
|
||||
minimax_m3_sparse_block_page_stride(k_cache, v_cache),
|
||||
)
|
||||
_run_gluon_decode(
|
||||
q,
|
||||
|
||||
@@ -25,6 +25,7 @@ class MiniMaxM3SparseAiterPAImpl(MiniMaxM3SparseImpl):
|
||||
output: torch.Tensor,
|
||||
*,
|
||||
query_fp8: torch.Tensor | None = None,
|
||||
decode_sparse_table: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.models.minimax_m3.amd.ops.sparse_pa import (
|
||||
minimax_m3_sparse_attn_decode_aiter,
|
||||
@@ -70,6 +71,12 @@ class MiniMaxM3SparseAiterPAImpl(MiniMaxM3SparseImpl):
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
decode_query_len=d.decode_query_len,
|
||||
sparse_block_table=(
|
||||
decode_sparse_table[0] if decode_sparse_table is not None else None
|
||||
),
|
||||
sparse_context_lens=(
|
||||
decode_sparse_table[1] if decode_sparse_table is not None else None
|
||||
),
|
||||
)
|
||||
|
||||
if main_md.num_prefills > 0:
|
||||
|
||||
@@ -369,6 +369,18 @@ class MiniMaxM3IndexerImpl(nn.Module):
|
||||
# Shared, stable-address top-k output buffer (set by the model for the
|
||||
# cudagraph-safe MSA impl); None -> impl allocates fresh (eager).
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
topk_completion_counter = None
|
||||
if current_platform.is_rocm() and topk_indices_buffer is not None:
|
||||
topk_completion_counter = torch.zeros(
|
||||
topk_indices_buffer.shape[:2],
|
||||
dtype=torch.int32,
|
||||
device=topk_indices_buffer.device,
|
||||
)
|
||||
self.register_buffer(
|
||||
"topk_completion_counter",
|
||||
topk_completion_counter,
|
||||
persistent=False,
|
||||
)
|
||||
# Owns the side cache (registers itself in the static forward context).
|
||||
self.index_cache = MiniMaxM3IndexerCache(
|
||||
head_dim=index_head_dim,
|
||||
@@ -392,6 +404,11 @@ class MiniMaxM3IndexerTritonImpl(MiniMaxM3IndexerImpl):
|
||||
def forward(
|
||||
self,
|
||||
index_query: torch.Tensor,
|
||||
*,
|
||||
attention_block_table: torch.Tensor | None = None,
|
||||
sparse_block_table_out: torch.Tensor | None = None,
|
||||
sparse_context_lens_out: torch.Tensor | None = None,
|
||||
block_page_stride: int | None = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
@@ -417,6 +434,19 @@ class MiniMaxM3IndexerTritonImpl(MiniMaxM3IndexerImpl):
|
||||
if index_md.num_decodes > 0:
|
||||
d = index_md.decode
|
||||
assert d is not None
|
||||
fused_sparse_kwargs = {}
|
||||
if attention_block_table is not None:
|
||||
fused_sparse_kwargs = {
|
||||
"attention_block_table": attention_block_table,
|
||||
"sparse_block_table_out": sparse_block_table_out,
|
||||
"sparse_context_lens_out": sparse_context_lens_out,
|
||||
"block_page_stride": block_page_stride,
|
||||
}
|
||||
decode_backend_kwargs = {}
|
||||
if current_platform.is_rocm():
|
||||
decode_backend_kwargs["completion_counter"] = (
|
||||
self.topk_completion_counter
|
||||
)
|
||||
decode_topk = minimax_m3_index_decode(
|
||||
iq[:nd],
|
||||
kv,
|
||||
@@ -430,6 +460,8 @@ class MiniMaxM3IndexerTritonImpl(MiniMaxM3IndexerImpl):
|
||||
d.decode_query_len,
|
||||
d.max_decode_query_len,
|
||||
out=buf_htk,
|
||||
**fused_sparse_kwargs,
|
||||
**decode_backend_kwargs,
|
||||
)
|
||||
if index_md.num_prefills > 0:
|
||||
p = index_md.prefill
|
||||
@@ -567,5 +599,43 @@ class MiniMaxM3Indexer(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
index_query: torch.Tensor,
|
||||
*,
|
||||
attention_block_table: torch.Tensor | None = None,
|
||||
sparse_block_table_out: torch.Tensor | None = None,
|
||||
sparse_context_lens_out: torch.Tensor | None = None,
|
||||
block_page_stride: int | None = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
return self.impl(index_query)
|
||||
if attention_block_table is None and all(
|
||||
arg is None
|
||||
for arg in (
|
||||
sparse_block_table_out,
|
||||
sparse_context_lens_out,
|
||||
block_page_stride,
|
||||
)
|
||||
):
|
||||
return self.impl(index_query)
|
||||
if attention_block_table is None or any(
|
||||
arg is None
|
||||
for arg in (
|
||||
sparse_block_table_out,
|
||||
sparse_context_lens_out,
|
||||
block_page_stride,
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"MiniMax-M3 fused decode sparse-table arguments must be "
|
||||
"provided together"
|
||||
)
|
||||
if not current_platform.is_rocm() or not isinstance(
|
||||
self.impl, MiniMaxM3IndexerTritonImpl
|
||||
):
|
||||
raise ValueError(
|
||||
"MiniMax-M3 fused decode sparse-table construction is ROCm-only"
|
||||
)
|
||||
return self.impl(
|
||||
index_query,
|
||||
attention_block_table=attention_block_table,
|
||||
sparse_block_table_out=sparse_block_table_out,
|
||||
sparse_context_lens_out=sparse_context_lens_out,
|
||||
block_page_stride=block_page_stride,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user