[ROCm][MiniMax-M3] Add AITER sparse paged attention (#47287)

Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Hongxia Yang <hongxia.yang@amd.com>
Co-authored-by: Jun Kang Chow <junkangchow@gmail.com>
Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
Tan Pin Siang
2026-07-12 19:27:29 -07:00
committed by GitHub
co-authored by vllmellm Hongxia Yang Jun Kang Chow tjtanaa
parent e26264f3ef
commit ee5a89f4d7
15 changed files with 1554 additions and 128 deletions
+214 -25
View File
@@ -35,6 +35,7 @@ from vllm.config import (
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention.attention import set_default_quant_scales
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
fused_allreduce_gemma_rms_norm,
@@ -93,6 +94,7 @@ from vllm.models.minimax_m3.common.mm_preprocess import (
from vllm.models.minimax_m3.common.sparse_attention import (
MiniMaxM3SparseBackend,
MiniMaxM3SparseImpl,
minimax_m3_use_aiter_sparse_pa,
select_main_impl_cls,
)
from vllm.models.minimax_m3.common.vision_tower import MiniMaxVLVisionModel
@@ -103,6 +105,7 @@ from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheSpec,
get_kv_quant_mode,
is_quantized_kv_cache,
)
@@ -636,6 +639,11 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
self.kv_cache_dtype, vllm_config.model_config
)
# MiniMax-M3 sparse attention owns its KV-cache insert/read path instead
# of wrapping the generic Attention module. Keep the same runtime scale
# attributes so FP8 KV reads can honor vLLM's per-layer descale contract.
self.calculate_kv_scales = False
set_default_quant_scales(self, register_buffer=True)
# Shared top-k buffer: the indexer writes the selected blocks into it and
# the attend impl reads them back (no Python value crosses the break).
@@ -647,6 +655,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
self.impl: MiniMaxM3SparseImpl = select_main_impl_cls( # type: ignore[assignment]
topk_blocks=sparse_cfg["sparse_topk_blocks"],
kv_cache_dtype=self.kv_cache_dtype,
num_kv_heads=self.num_kv_heads,
)(
self.num_heads,
self.head_dim,
@@ -656,6 +665,10 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
topk_blocks=sparse_cfg["sparse_topk_blocks"],
sparse_block_size=sparse_cfg["sparse_block_size"],
)
self.use_aiter_sparse_pa = minimax_m3_use_aiter_sparse_pa(self.num_kv_heads)
self.kv_cache_k = torch.tensor([])
self.kv_cache_v = torch.tensor([])
self._aiter_sparse_pa_cache_data_ptr = 0
# Self-contained nn.Module: owns its side cache, selects its impl in init
# (Triton on ROCm, where the SM100 gate is always False).
self.indexer = MiniMaxM3Indexer(
@@ -694,6 +707,91 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
)
def _ensure_aiter_sparse_pa_kv_cache(self) -> None:
if self.kv_cache.numel() == 0:
return
if self._aiter_sparse_pa_cache_data_ptr == self.kv_cache.data_ptr():
return
kv_cache = self.kv_cache
if is_quantized_kv_cache(self.kv_cache_dtype):
kv_cache = kv_cache.view(self.impl.kv_cache_fp8_dtype)
key_cache, value_cache = kv_cache.unbind(1)
if not key_cache.is_contiguous() or not value_cache.is_contiguous():
raise RuntimeError(
"MiniMax-M3 AITER sparse PA requires K/V-separated KV cache "
"storage. Set VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 before "
"initializing the engine."
)
x = 16 // key_cache.element_size()
if self.head_dim % x != 0:
raise RuntimeError(
"MiniMax-M3 AITER sparse PA requires head_dim divisible by "
f"16 / dtype_size, got head_dim={self.head_dim}, x={x}"
)
num_blocks = key_cache.shape[0]
num_phys16 = num_blocks * 8
self.kv_cache_k = key_cache.view(
num_phys16,
self.num_kv_heads,
self.head_dim // x,
16,
x,
)
self.kv_cache_v = value_cache.view(
num_phys16,
self.num_kv_heads,
16 // x,
self.head_dim,
x,
)
self._aiter_sparse_pa_cache_data_ptr = self.kv_cache.data_ptr()
def get_aiter_sparse_pa_kv_cache(self) -> tuple[torch.Tensor, torch.Tensor]:
self._ensure_aiter_sparse_pa_kv_cache()
return self.kv_cache_k, self.kv_cache_v
def _insert_aiter_sparse_pa_kv(
self,
k: torch.Tensor,
v: torch.Tensor,
index_k: torch.Tensor | None,
slot_mapping: torch.Tensor,
index_slot_mapping: torch.Tensor | None,
) -> None:
if self.kv_cache.numel() == 0:
return
from aiter import reshape_and_cache
from vllm.models.minimax_m3.amd.ops.sparse_pa import (
minimax_m3_insert_index_cache,
)
key_cache, value_cache = self.get_aiter_sparse_pa_kv_cache()
kv_cache_dtype = (
self.kv_cache_dtype
if is_quantized_kv_cache(self.kv_cache_dtype)
else "auto"
)
reshape_and_cache(
k.contiguous(),
v.contiguous(),
key_cache,
value_cache,
slot_mapping,
kv_cache_dtype=kv_cache_dtype,
k_scale=getattr(self, "_k_scale", None),
v_scale=getattr(self, "_v_scale", None),
asm_layout=True,
)
if index_k is None or index_slot_mapping is None:
return
index_cache = self.indexer.index_cache.kv_cache
if index_cache.numel() == 0:
return
minimax_m3_insert_index_cache(index_k, index_cache, index_slot_mapping)
def forward(
self,
positions: torch.Tensor,
@@ -724,31 +822,121 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
return qkv.new_zeros((num_tokens, self.hidden_size))
main_slot_mapping = fwd_slot_mapping[self.layer_name]
index_slot_mapping = fwd_slot_mapping[self.indexer.index_cache.prefix]
q = qkv.new_empty((num_tokens, self.q_size))
index_q = qkv.new_empty((num_tokens, self.index_q_size))
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
self.q_norm.weight,
self.k_norm.weight,
cos_sin_cache,
positions,
self.num_heads,
self.num_kv_heads,
rotary_dim,
eps,
self.index_q_norm.weight,
self.index_k_norm.weight,
self.num_idx_heads,
main_slot_mapping,
index_slot_mapping,
self.kv_cache,
self.indexer.index_cache.kv_cache,
self.kv_cache.size(2), # paged-cache block size
q,
index_q,
self.kv_cache_dtype,
)
if self.skip_index_topk:
index_q = None
if self.use_aiter_sparse_pa:
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
self.q_norm.weight,
self.k_norm.weight,
cos_sin_cache,
positions,
self.num_heads,
self.num_kv_heads,
rotary_dim,
eps,
num_index_heads=self.num_idx_heads,
q_out=q,
skip_index_branch=True,
)
k_start = self.q_size
v_start = k_start + self.kv_size
k = qkv[:, k_start:v_start].view(
num_tokens, self.num_kv_heads, self.head_dim
)
v = qkv[:, v_start : v_start + self.kv_size].view(
num_tokens, self.num_kv_heads, self.head_dim
)
self._insert_aiter_sparse_pa_kv(
k,
v,
None,
main_slot_mapping,
None,
)
else:
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
self.q_norm.weight,
self.k_norm.weight,
cos_sin_cache,
positions,
self.num_heads,
self.num_kv_heads,
rotary_dim,
eps,
num_index_heads=self.num_idx_heads,
slot_mapping=main_slot_mapping,
kv_cache=self.kv_cache,
block_size=self.kv_cache.size(2), # paged-cache block size
q_out=q,
kv_cache_dtype=self.kv_cache_dtype,
skip_index_branch=True,
)
else:
index_slot_mapping = fwd_slot_mapping[self.indexer.index_cache.prefix]
index_q = qkv.new_empty((num_tokens, self.index_q_size))
if self.use_aiter_sparse_pa:
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
self.q_norm.weight,
self.k_norm.weight,
cos_sin_cache,
positions,
self.num_heads,
self.num_kv_heads,
rotary_dim,
eps,
self.index_q_norm.weight,
self.index_k_norm.weight,
self.num_idx_heads,
q_out=q,
index_q_out=index_q,
kv_cache_dtype=self.kv_cache_dtype,
)
k_start = self.q_size
v_start = k_start + self.kv_size
index_k_start = v_start + self.kv_size + self.index_q_size
k = qkv[:, k_start:v_start].view(
num_tokens, self.num_kv_heads, self.head_dim
)
v = qkv[:, v_start : v_start + self.kv_size].view(
num_tokens, self.num_kv_heads, self.head_dim
)
index_k = qkv[
:, index_k_start : index_k_start + self.idx_head_dim
].view(num_tokens, self.idx_head_dim)
self._insert_aiter_sparse_pa_kv(
k,
v,
index_k,
main_slot_mapping,
index_slot_mapping,
)
else:
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
self.q_norm.weight,
self.k_norm.weight,
cos_sin_cache,
positions,
self.num_heads,
self.num_kv_heads,
rotary_dim,
eps,
self.index_q_norm.weight,
self.index_k_norm.weight,
self.num_idx_heads,
main_slot_mapping,
index_slot_mapping,
self.kv_cache,
self.indexer.index_cache.kv_cache,
self.kv_cache.size(2), # paged-cache block size
q,
index_q,
self.kv_cache_dtype,
)
output = torch.empty_like(q)
attn_output = self._run_attention(q, index_q, output)
@@ -759,7 +947,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
def _run_attention(
self,
query: torch.Tensor,
index_query: torch.Tensor,
index_query: torch.Tensor | None,
output: torch.Tensor,
) -> torch.Tensor:
# Single eager break around both: their split-K kernels read per-request
@@ -768,6 +956,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
# When skip_index_topk is set (ATOM index_topk_freq), reuse the selection
# the preceding compute layer wrote into the shared buffer this forward.
if not self.skip_index_topk:
assert index_query is not None
self.indexer(index_query)
return self.impl.forward(self, query, self.kv_cache, output)
+65 -8
View File
@@ -12,7 +12,9 @@ import torch
from vllm.models.minimax_m3.common.ops.sparse_attn import (
_FP8_DTYPES,
_KV_SCALE_NONE,
SPARSE_BLOCK_SIZE,
_kv_scale_args,
minimax_m3_sparse_attn_decode,
)
from vllm.platforms.rocm import on_gfx950, on_mi3xx
@@ -70,7 +72,9 @@ def _sparse_attn_prefill_kwargs() -> dict:
@triton.jit(do_not_specialize_on_alignment=["seq_lens", "prefix_lens"])
def _gqa_sparse_fwd_kernel(
q_ptr, # [total_q, num_heads, head_dim]
kv_cache_ptr, # main cache: [num_blocks, 2, 128, num_kv_heads, head_dim]
kv_cache_ptr, # main cache: [num_blocks, num_kv_heads, 128, 2*head_dim]
k_scale_ptr,
v_scale_ptr,
t_ptr, # topk_idx: [num_kv_heads, total_q, topk]
o_ptr, # [total_q, num_heads, head_dim]
block_table_ptr, # [num_reqs, max_blocks]
@@ -88,10 +92,13 @@ def _gqa_sparse_fwd_kernel(
stride_qh,
stride_qd,
stride_kv_blk,
stride_kv_kv,
stride_kv_pos,
stride_kv_h,
stride_kv_pos,
stride_kv_d,
stride_ks_h,
stride_ks_t,
stride_vs_h,
stride_vs_t,
stride_th,
stride_tn,
stride_tk,
@@ -106,6 +113,7 @@ def _gqa_sparse_fwd_kernel(
BLOCK_SIZE_T: tl.constexpr,
BLOCK_SIZE_QH: tl.constexpr,
USE_FP8: tl.constexpr, # fp8 KV cache: dequantize K/V to q.dtype on load
KV_SCALE_MODE: tl.constexpr, # 0: none, 1: scalar, 2: [kv_head, token]
SUB_K: tl.constexpr, # CDNA only: KV sub-tile width (see _IS_MI3XX)
):
sm_scale_log2e = sm_scale * 1.4426950409
@@ -161,7 +169,6 @@ def _gqa_sparse_fwd_kernel(
pos_mask_sub = pos_sub < seq_len
k_sub = tl.load(
kv_base
+ 0 * stride_kv_kv
+ off_sub[None, :] * stride_kv_pos
+ off_d[:, None] * stride_kv_d,
mask=d_mask[:, None] & pos_mask_sub[None, :],
@@ -169,6 +176,17 @@ def _gqa_sparse_fwd_kernel(
)
if USE_FP8:
k_sub = k_sub.to(q.dtype)
if KV_SCALE_MODE == 1:
k_sub = (k_sub * tl.load(k_scale_ptr)).to(q.dtype)
elif KV_SCALE_MODE == 2:
k_scale = tl.load(
k_scale_ptr
+ pid_kh * stride_ks_h
+ (page * BLOCK_SIZE_K + off_sub) * stride_ks_t,
mask=pos_mask_sub,
other=1.0,
)
k_sub = (k_sub * k_scale[None, :]).to(q.dtype)
off_q_sub = (
tl.arange(0, BLOCK_SIZE_Q)[:, None]
+ pid_q_j * BLOCK_SIZE_Q
@@ -187,14 +205,24 @@ def _gqa_sparse_fwd_kernel(
acc_o = acc_o * tl.exp2(m_i - m_ij)[:, None]
v_sub = tl.load(
kv_base
+ 1 * stride_kv_kv
+ off_sub[:, None] * stride_kv_pos
+ off_d[None, :] * stride_kv_d,
+ (head_dim + off_d[None, :]) * stride_kv_d,
mask=pos_mask_sub[:, None] & d_mask[None, :],
other=0.0,
)
if USE_FP8:
v_sub = v_sub.to(q.dtype)
if KV_SCALE_MODE == 1:
v_sub = (v_sub * tl.load(v_scale_ptr)).to(q.dtype)
elif KV_SCALE_MODE == 2:
v_scale = tl.load(
v_scale_ptr
+ pid_kh * stride_vs_h
+ (page * BLOCK_SIZE_K + off_sub) * stride_vs_t,
mask=pos_mask_sub,
other=1.0,
)
v_sub = (v_sub * v_scale[:, None]).to(q.dtype)
acc_o += tl.dot(p_sub.to(v_sub.dtype), v_sub)
m_i = m_ij
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
@@ -214,7 +242,7 @@ def _gqa_sparse_fwd_kernel(
@torch.no_grad()
def minimax_m3_sparse_attn(
q: torch.Tensor, # [total_q, num_heads, head_dim]
kv_cache: torch.Tensor, # [num_blocks, 2, 128, num_kv_heads, head_dim]
kv_cache: torch.Tensor, # [num_blocks, num_kv_heads, 128, 2*head_dim]
topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk]
block_table: torch.Tensor, # [batch, max_blocks]
cu_seqlens_q: torch.Tensor, # [batch+1] int32
@@ -224,6 +252,8 @@ def minimax_m3_sparse_attn(
num_kv_heads: int,
sm_scale: float,
output: torch.Tensor, # [total_q, num_heads, head_dim]
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
) -> None:
"""GQA block-sparse attention over the selected blocks. block_size_q == 1."""
total_q, num_heads, head_dim = q.shape
@@ -231,10 +261,33 @@ def minimax_m3_sparse_attn(
topk = topk_idx.shape[-1]
gqa_group_size = num_heads // num_kv_heads
use_fp8 = kv_cache.dtype in _FP8_DTYPES
(
k_scale_arg,
v_scale_arg,
stride_ks_h,
stride_ks_t,
stride_vs_h,
stride_vs_t,
kv_scale_mode,
) = (
_kv_scale_args(output, num_kv_heads, k_scale, v_scale)
if use_fp8
else (
output,
output,
0,
0,
0,
0,
_KV_SCALE_NONE,
)
)
grid = (max_query_len, num_kv_heads, batch)
_gqa_sparse_fwd_kernel[grid](
q,
kv_cache,
k_scale_arg,
v_scale_arg,
topk_idx,
output,
block_table,
@@ -255,7 +308,10 @@ def minimax_m3_sparse_attn(
kv_cache.stride(1),
kv_cache.stride(2),
kv_cache.stride(3),
kv_cache.stride(4),
stride_ks_h,
stride_ks_t,
stride_vs_h,
stride_vs_t,
topk_idx.stride(0),
topk_idx.stride(1),
topk_idx.stride(2),
@@ -266,6 +322,7 @@ def minimax_m3_sparse_attn(
BLOCK_SIZE_Q=1,
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
USE_FP8=use_fp8,
KV_SCALE_MODE=kv_scale_mode,
SUB_K=_SPARSE_ATTN_SUB_K,
**_sparse_attn_prefill_kwargs(),
)
+466
View File
@@ -0,0 +1,466 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""AITER page-16 sparse paged-attention helpers for MiniMax-M3 on ROCm."""
import torch
try:
from vllm.triton_utils import tl, triton
except ModuleNotFoundError:
import triton
import triton.language as tl
from vllm.models.minimax_m3.common.ops.sparse_attn import SPARSE_BLOCK_SIZE
ASM_PAGE_SIZE = 16
PAGES_PER_SPARSE_BLOCK = SPARSE_BLOCK_SIZE // ASM_PAGE_SIZE
_FP8_DTYPES = {
dtype
for dtype in (
getattr(torch, "float8_e4m3fn", None),
getattr(torch, "float8_e4m3fnuz", None),
getattr(torch, "float8_e5m2", None),
getattr(torch, "float8_e5m2fnuz", None),
)
if dtype is not None
}
def _is_fp8_kv_cache_tensor(kv_cache: torch.Tensor) -> bool:
return kv_cache.dtype in _FP8_DTYPES
@triton.jit
def _build_sparse_block_table_kernel(
topk_ptr, # [1, batch, topk] int32, selected logical 128-block ids
block_table_ptr, # [batch, max_blocks] int32, logical 128-page table
seq_lens_ptr, # [batch] int32
sparse_bt_ptr, # [batch, topk * 8] int32, physical 16-page table
sparse_ctx_ptr, # [batch] int32
max_topk,
stride_topk_n,
stride_topk_k,
stride_bt_b,
stride_sbt_b,
SPARSE_BLOCK_SIZE_C: tl.constexpr,
PAGES_PER_BLOCK: tl.constexpr,
BLOCK_SIZE_T: tl.constexpr,
):
pid_b = tl.program_id(0)
seq_len = tl.load(seq_lens_ptr + pid_b)
last_blk = (seq_len - 1) // SPARSE_BLOCK_SIZE_C
topk_row = topk_ptr + pid_b * stride_topk_n
bt_row = block_table_ptr + pid_b * stride_bt_b
sbt_row = sparse_bt_ptr + pid_b * stride_sbt_b
off_t = tl.arange(0, BLOCK_SIZE_T)
blk = tl.load(topk_row + off_t * stride_topk_k, mask=off_t < max_topk, other=-1)
valid = blk >= 0
is_tail = valid & (blk == last_blk)
is_full = valid & (blk != last_blk)
n_full = tl.sum(is_full.to(tl.int32), axis=0)
n_valid = tl.sum(valid.to(tl.int32), axis=0)
earlier_full = tl.cumsum(is_full.to(tl.int32), axis=0) - is_full.to(tl.int32)
slot = tl.where(is_full, earlier_full, n_full)
logical_page = tl.load(bt_row + blk, mask=valid, other=0).to(tl.int32)
base_phys = logical_page * PAGES_PER_BLOCK
dst_base = slot * PAGES_PER_BLOCK
for j in tl.static_range(PAGES_PER_BLOCK):
tl.store(sbt_row + dst_base + j, base_phys + j, mask=valid)
n_used = n_valid * PAGES_PER_BLOCK
off_w = tl.arange(0, BLOCK_SIZE_T * PAGES_PER_BLOCK)
row_width = max_topk * PAGES_PER_BLOCK
tl.store(
sbt_row + off_w,
tl.zeros_like(off_w),
mask=(off_w >= n_used) & (off_w < row_width),
)
tail_tokens = seq_len - last_blk * SPARSE_BLOCK_SIZE_C
has_tail = tl.sum(is_tail.to(tl.int32), axis=0) > 0
ctx = n_full * SPARSE_BLOCK_SIZE_C + tl.where(has_tail, tail_tokens, 0)
ctx = tl.where(has_tail, ctx, tl.minimum(n_valid * SPARSE_BLOCK_SIZE_C, seq_len))
tl.store(sparse_ctx_ptr + pid_b, ctx)
@torch.no_grad()
def minimax_m3_build_sparse_block_table(
topk_idx: torch.Tensor, # [1, batch, topk]
block_table: torch.Tensor, # [batch, max_blocks]
seq_lens: torch.Tensor, # [batch]
) -> tuple[torch.Tensor, torch.Tensor]:
"""Compact selected logical sparse blocks into physical page-16 tables."""
assert topk_idx.shape[0] == 1, "AITER sparse PA requires num_kv_heads == 1"
batch = topk_idx.shape[1]
topk = topk_idx.shape[-1]
width = topk * PAGES_PER_SPARSE_BLOCK
sparse_bt = torch.empty((batch, width), dtype=torch.int32, device=topk_idx.device)
sparse_ctx = torch.empty((batch,), dtype=torch.int32, device=topk_idx.device)
_build_sparse_block_table_kernel[(batch,)](
topk_idx,
block_table,
seq_lens,
sparse_bt,
sparse_ctx,
topk,
topk_idx.stride(1),
topk_idx.stride(2),
block_table.stride(0),
sparse_bt.stride(0),
SPARSE_BLOCK_SIZE_C=SPARSE_BLOCK_SIZE,
PAGES_PER_BLOCK=PAGES_PER_SPARSE_BLOCK,
BLOCK_SIZE_T=triton.next_power_of_2(topk),
)
return sparse_bt, sparse_ctx
@triton.jit
def _build_sparse_block_table_prefill_kernel(
topk_ptr, # [1, total_q, topk]
block_table_ptr, # [batch, max_blocks]
req_id_ptr, # [total_q]
abs_pos_ptr, # [total_q]
sparse_bt_ptr, # [total_q, topk * 8]
sparse_ctx_ptr, # [total_q]
max_topk,
stride_topk_n,
stride_topk_k,
stride_bt_b,
stride_sbt_n,
SPARSE_BLOCK_SIZE_C: tl.constexpr,
PAGES_PER_BLOCK: tl.constexpr,
BLOCK_SIZE_T: tl.constexpr,
):
pid_n = tl.program_id(0)
req_id = tl.load(req_id_ptr + pid_n)
abs_pos = tl.load(abs_pos_ptr + pid_n)
causal_len = abs_pos + 1
self_blk = abs_pos // SPARSE_BLOCK_SIZE_C
topk_row = topk_ptr + pid_n * stride_topk_n
bt_row = block_table_ptr + req_id * stride_bt_b
sbt_row = sparse_bt_ptr + pid_n * stride_sbt_n
off_t = tl.arange(0, BLOCK_SIZE_T)
blk = tl.load(topk_row + off_t * stride_topk_k, mask=off_t < max_topk, other=-1)
valid = (blk >= 0) & (blk <= self_blk)
is_tail = valid & (blk == self_blk)
is_full = valid & (blk < self_blk)
n_full = tl.sum(is_full.to(tl.int32), axis=0)
n_valid = tl.sum(valid.to(tl.int32), axis=0)
earlier_full = tl.cumsum(is_full.to(tl.int32), axis=0) - is_full.to(tl.int32)
slot = tl.where(is_full, earlier_full, n_full)
logical_page = tl.load(bt_row + blk, mask=valid, other=0).to(tl.int32)
base_phys = logical_page * PAGES_PER_BLOCK
dst_base = slot * PAGES_PER_BLOCK
for j in tl.static_range(PAGES_PER_BLOCK):
tl.store(sbt_row + dst_base + j, base_phys + j, mask=valid)
n_used = n_valid * PAGES_PER_BLOCK
off_w = tl.arange(0, BLOCK_SIZE_T * PAGES_PER_BLOCK)
row_width = max_topk * PAGES_PER_BLOCK
tl.store(
sbt_row + off_w,
tl.zeros_like(off_w),
mask=(off_w >= n_used) & (off_w < row_width),
)
tail_tokens = causal_len - self_blk * SPARSE_BLOCK_SIZE_C
has_tail = tl.sum(is_tail.to(tl.int32), axis=0) > 0
ctx = n_full * SPARSE_BLOCK_SIZE_C + tl.where(has_tail, tail_tokens, 0)
ctx = tl.where(has_tail, ctx, tl.minimum(n_valid * SPARSE_BLOCK_SIZE_C, causal_len))
tl.store(sparse_ctx_ptr + pid_n, ctx)
@torch.no_grad()
def minimax_m3_build_sparse_block_table_prefill(
topk_idx: torch.Tensor, # [1, total_q, topk]
block_table: torch.Tensor, # [batch, max_blocks]
query_req_id: torch.Tensor, # [total_q]
query_abs_pos: torch.Tensor, # [total_q]
) -> tuple[torch.Tensor, torch.Tensor]:
"""Build one page-16 sparse block table row per prefill query token."""
assert topk_idx.shape[0] == 1, "AITER sparse PA requires num_kv_heads == 1"
total_q = topk_idx.shape[1]
topk = topk_idx.shape[-1]
width = topk * PAGES_PER_SPARSE_BLOCK
sparse_bt = torch.empty((total_q, width), dtype=torch.int32, device=topk_idx.device)
sparse_ctx = torch.empty((total_q,), dtype=torch.int32, device=topk_idx.device)
_build_sparse_block_table_prefill_kernel[(total_q,)](
topk_idx,
block_table,
query_req_id,
query_abs_pos,
sparse_bt,
sparse_ctx,
topk,
topk_idx.stride(1),
topk_idx.stride(2),
block_table.stride(0),
sparse_bt.stride(0),
SPARSE_BLOCK_SIZE_C=SPARSE_BLOCK_SIZE,
PAGES_PER_BLOCK=PAGES_PER_SPARSE_BLOCK,
BLOCK_SIZE_T=triton.next_power_of_2(topk),
)
return sparse_bt, sparse_ctx
@triton.jit
def _insert_index_cache_kernel(
index_k_ptr, # [num_tokens, head_dim]
index_cache_ptr, # [num_blocks, block_size, head_dim]
slot_mapping_ptr, # [num_tokens]
stride_k_t,
stride_k_d,
stride_c_b,
stride_c_t,
stride_c_d,
stride_slot_t,
CACHE_BLOCK_SIZE: tl.constexpr,
HEAD_DIM: tl.constexpr,
BLOCK_D: tl.constexpr,
):
pid_t = tl.program_id(0)
offs_d = tl.arange(0, BLOCK_D)
slot = tl.load(slot_mapping_ptr + pid_t * stride_slot_t)
block_id = slot // CACHE_BLOCK_SIZE
block_off = slot - block_id * CACHE_BLOCK_SIZE
src = index_k_ptr + pid_t * stride_k_t + offs_d * stride_k_d
dst = (
index_cache_ptr
+ block_id * stride_c_b
+ block_off * stride_c_t
+ offs_d * stride_c_d
)
mask = (slot >= 0) & (offs_d < HEAD_DIM)
value = tl.load(src, mask=offs_d < HEAD_DIM, other=0.0)
tl.store(dst, value, mask=mask)
@torch.no_grad()
def minimax_m3_insert_index_cache(
index_k: torch.Tensor,
index_cache: torch.Tensor,
index_slot_mapping: torch.Tensor,
) -> None:
"""Scatter index keys into MiniMax-M3's key-only side cache."""
if index_k.numel() == 0 or index_cache.numel() == 0:
return
if index_k.dim() != 2 or index_cache.dim() != 3:
raise ValueError("MiniMax-M3 index cache insert expects [N,D] and [B,T,D]")
if index_k.shape[1] != index_cache.shape[2]:
raise ValueError("MiniMax-M3 index key dim must match index cache head dim")
if index_slot_mapping.dim() != 1 or index_slot_mapping.shape[0] != index_k.shape[0]:
raise ValueError("MiniMax-M3 index slot mapping must be a length-N vector")
if index_cache.stride(2) != 1:
raise ValueError("MiniMax-M3 index cache requires contiguous head dimension")
head_dim = index_k.shape[1]
_insert_index_cache_kernel[(index_k.shape[0],)](
index_k,
index_cache,
index_slot_mapping,
index_k.stride(0),
index_k.stride(1),
index_cache.stride(0),
index_cache.stride(1),
index_cache.stride(2),
index_slot_mapping.stride(0),
CACHE_BLOCK_SIZE=index_cache.shape[1],
HEAD_DIM=head_dim,
BLOCK_D=triton.next_power_of_2(head_dim),
num_warps=4,
)
def _gluon_scale_arg(
scale: torch.Tensor | None,
*,
num_phys_pages: int,
num_kv_heads: int,
) -> torch.Tensor | None:
if scale is None:
return None
if scale.numel() == 1:
return scale
if scale.dim() != 2 or scale.shape[0] != num_kv_heads:
raise ValueError(
"MiniMax-M3 AITER sparse PA supports scalar KV scales or "
"[num_kv_heads, max_kv_tokens] scales"
)
max_tokens = num_phys_pages * ASM_PAGE_SIZE
if scale.shape[1] < max_tokens:
raise ValueError(
"MiniMax-M3 AITER sparse PA KV scale tensor is smaller than the "
f"cache token capacity ({scale.shape[1]} < {max_tokens})"
)
scale = scale[:, :max_tokens]
return (
scale.transpose(0, 1)
.contiguous()
.view(num_phys_pages, ASM_PAGE_SIZE, num_kv_heads)
.permute(0, 2, 1)
.contiguous()
.unsqueeze(-1)
)
@torch.no_grad()
def minimax_m3_sparse_attn_decode_aiter(
q: torch.Tensor, # [batch, num_heads, head_dim]
k_cache: torch.Tensor, # [phys16, num_kv_heads, head_dim // x, 16, x]
v_cache: torch.Tensor, # [phys16, num_kv_heads, 16 // x, head_dim, x]
topk_idx: torch.Tensor, # [1, batch, topk]
block_table: torch.Tensor, # [batch, max_blocks]
seq_lens: torch.Tensor, # [batch]
num_kv_heads: int,
sm_scale: float,
output: torch.Tensor, # [batch, num_heads, head_dim]
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
) -> None:
sparse_bt, sparse_ctx = minimax_m3_build_sparse_block_table(
topk_idx, block_table, seq_lens
)
_run_gluon_decode(
q,
k_cache,
v_cache,
sparse_bt,
sparse_ctx,
num_kv_heads,
sm_scale,
output,
k_scale,
v_scale,
)
@torch.no_grad()
def minimax_m3_sparse_attn_prefill_aiter(
q: torch.Tensor, # [total_q, num_heads, head_dim]
k_cache: torch.Tensor,
v_cache: torch.Tensor,
topk_idx: torch.Tensor, # [1, total_q, topk]
block_table: torch.Tensor,
cu_seqlens_q: torch.Tensor,
prefix_lens: torch.Tensor,
num_kv_heads: int,
sm_scale: float,
output: torch.Tensor,
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
) -> None:
total_q = q.shape[0]
pos = torch.arange(total_q, dtype=torch.int32, device=q.device)
query_req_id = torch.searchsorted(cu_seqlens_q[1:].contiguous(), pos, right=True)
query_req_id = query_req_id.to(torch.int32)
query_abs_pos = (prefix_lens[query_req_id] + (pos - cu_seqlens_q[query_req_id])).to(
torch.int32
)
sparse_bt, sparse_ctx = minimax_m3_build_sparse_block_table_prefill(
topk_idx, block_table, query_req_id, query_abs_pos
)
_run_gluon_decode(
q,
k_cache,
v_cache,
sparse_bt,
sparse_ctx,
num_kv_heads,
sm_scale,
output,
k_scale,
v_scale,
)
def _run_gluon_decode(
q: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
sparse_bt: torch.Tensor,
sparse_ctx: torch.Tensor,
num_kv_heads: int,
sm_scale: float,
output: torch.Tensor,
k_scale: torch.Tensor | None,
v_scale: torch.Tensor | None,
) -> None:
from aiter import dtypes as aiter_dtypes
from aiter.ops.triton.gluon.pa_decode_gluon import (
get_recommended_splits,
pa_decode_gluon,
)
if not q.is_contiguous():
q = q.contiguous()
if not output.is_contiguous():
raise ValueError("MiniMax-M3 AITER sparse PA output must be contiguous")
total_q, num_q_heads, head_size = q.shape
if head_size != 128:
raise ValueError("MiniMax-M3 AITER sparse PA requires head_dim == 128")
group_size = num_q_heads // num_kv_heads
nphys16, hkv = k_cache.shape[0], k_cache.shape[1]
k_cache_view = k_cache.view(nphys16 * hkv, 1, *k_cache.shape[2:])
v_cache_view = v_cache.view(nphys16 * hkv, 1, *v_cache.shape[2:])
q_view = q.view(total_q * num_kv_heads, group_size, head_size)
out_view = output.view(total_q * num_kv_heads, group_size, head_size)
num_seqs = total_q * num_kv_heads
max_context_partition_num = get_recommended_splits(num_seqs, 1)
context_partition_size = 256
intermediate_shape = (num_seqs, 1, max_context_partition_num, group_size)
exp_sums = torch.empty(intermediate_shape, dtype=torch.float32, device=q.device)
max_logits = torch.empty_like(exp_sums)
temporary_output = torch.empty(
*intermediate_shape, head_size, dtype=q.dtype, device=q.device
)
is_fp8 = _is_fp8_kv_cache_tensor(k_cache)
compute_type = aiter_dtypes.fp8 if is_fp8 else q.dtype
if is_fp8:
k_scale_arg = _gluon_scale_arg(
k_scale, num_phys_pages=nphys16, num_kv_heads=hkv
)
v_scale_arg = _gluon_scale_arg(
v_scale, num_phys_pages=nphys16, num_kv_heads=hkv
)
else:
k_scale_arg = v_scale_arg = None
pa_decode_gluon(
output=out_view,
query=q_view,
key_cache=k_cache_view,
value_cache=v_cache_view,
context_lengths=sparse_ctx,
block_tables=sparse_bt,
softmax_scale=sm_scale,
query_length=1,
max_context_partition_num=max_context_partition_num,
context_partition_size=context_partition_size,
compute_type=compute_type,
query_scale=None,
key_scale=k_scale_arg,
value_scale=v_scale_arg,
exp_sums=exp_sums,
max_logits=max_logits,
temporary_output=temporary_output,
alibi_slopes=None,
sinks=None,
sliding_window=-1,
ps=True,
)
@@ -0,0 +1,94 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MSA AITER block-sparse attend for MiniMax M3."""
import torch
from vllm.forward_context import get_forward_context
from vllm.models.minimax_m3.common.sparse_attention import (
MiniMaxM3SparseImpl,
MiniMaxM3SparseMetadata,
)
from vllm.v1.attention.backend import (
AttentionLayer,
)
class MiniMaxM3SparseAiterPAImpl(MiniMaxM3SparseImpl):
"""ROCm AITER page-16 SHUFFLE sparse paged attention."""
def forward(
self,
layer: AttentionLayer,
query: torch.Tensor,
kv_cache: torch.Tensor,
output: torch.Tensor,
) -> torch.Tensor:
from vllm.models.minimax_m3.amd.ops.sparse_pa import (
minimax_m3_sparse_attn_decode_aiter,
minimax_m3_sparse_attn_prefill_aiter,
)
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
return output
main_md = attn_metadata[layer.layer_name] # type: ignore[attr-defined]
assert isinstance(main_md, MiniMaxM3SparseMetadata)
nd = main_md.num_decode_tokens
num_tokens = main_md.num_actual_tokens
topk = layer.topk_indices_buffer # type: ignore[attr-defined]
assert topk is not None
if self.num_kv_heads != 1:
raise NotImplementedError(
"MiniMax-M3 AITER sparse PA currently requires per-rank "
f"num_kv_heads == 1, got {self.num_kv_heads}"
)
hd = self.head_size
q = query[:num_tokens].view(-1, self.num_heads, hd)
out = output[:num_tokens].view(-1, self.num_heads, hd)
k_cache, v_cache = layer.get_aiter_sparse_pa_kv_cache() # type: ignore[attr-defined]
k_scale = getattr(layer, "_k_scale", None) if self.use_fp8_kv else None
v_scale = getattr(layer, "_v_scale", None) if self.use_fp8_kv else None
if main_md.num_decodes > 0:
d = main_md.decode
assert d is not None
if d.decode_query_len != 1:
raise NotImplementedError(
"MiniMax-M3 AITER sparse PA does not support speculative "
f"decode_query_len={d.decode_query_len}"
)
minimax_m3_sparse_attn_decode_aiter(
q[:nd],
k_cache,
v_cache,
topk[:, :nd, :],
d.block_table,
d.seq_lens,
self.num_kv_heads,
self.scale,
out[:nd],
k_scale=k_scale,
v_scale=v_scale,
)
if main_md.num_prefills > 0:
p = main_md.prefill
assert p is not None
minimax_m3_sparse_attn_prefill_aiter(
q[nd:],
k_cache,
v_cache,
topk[:, nd:num_tokens, :],
p.block_table,
p.cu_seqlens_q,
p.context_lens,
self.num_kv_heads,
self.scale,
out[nd:],
k_scale=k_scale,
v_scale=v_scale,
)
return output
@@ -52,6 +52,8 @@ _FP8_DTYPES = (
def _gqa_sparse_fwd_kernel(
q_ptr, # [total_q, num_heads, head_dim]
kv_cache_ptr, # main cache: [num_blocks, num_kv_heads, 128, 2*head_dim]
k_scale_ptr,
v_scale_ptr,
t_ptr, # topk_idx: [num_kv_heads, total_q, topk]
o_ptr, # [total_q, num_heads, head_dim]
block_table_ptr, # [num_reqs, max_blocks]
@@ -72,6 +74,10 @@ def _gqa_sparse_fwd_kernel(
stride_kv_h,
stride_kv_pos,
stride_kv_d,
stride_ks_h,
stride_ks_t,
stride_vs_h,
stride_vs_t,
stride_th,
stride_tn,
stride_tk,
@@ -85,6 +91,7 @@ def _gqa_sparse_fwd_kernel(
BLOCK_SIZE_H: tl.constexpr,
BLOCK_SIZE_QH: tl.constexpr,
USE_FP8: tl.constexpr, # fp8 KV cache: dequantize K/V to q.dtype on load
KV_SCALE_MODE: tl.constexpr, # 0: none, 1: scalar, 2: [kv_head, token]
):
sm_scale_log2e = sm_scale * 1.4426950409
pid_q = tl.program_id(0)
@@ -148,6 +155,17 @@ def _gqa_sparse_fwd_kernel(
)
if USE_FP8:
k = k.to(q.dtype)
if KV_SCALE_MODE == 1:
k = (k * tl.load(k_scale_ptr)).to(q.dtype)
elif KV_SCALE_MODE == 2:
k_scale = tl.load(
k_scale_ptr
+ pid_kh * stride_ks_h
+ (page * BLOCK_SIZE_K + off_n) * stride_ks_t,
mask=pos_mask,
other=1.0,
)
k = (k * k_scale[None, :]).to(q.dtype)
qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
# causal: q_abs_pos - k_off >= block_start (c)
qk += tl.where(off_q[:, None, :] >= c, 0, float("-inf"))
@@ -169,6 +187,17 @@ def _gqa_sparse_fwd_kernel(
)
if USE_FP8:
v = v.to(q.dtype)
if KV_SCALE_MODE == 1:
v = (v * tl.load(v_scale_ptr)).to(q.dtype)
elif KV_SCALE_MODE == 2:
v_scale = tl.load(
v_scale_ptr
+ pid_kh * stride_vs_h
+ (page * BLOCK_SIZE_K + off_n) * stride_vs_t,
mask=pos_mask,
other=1.0,
)
v = (v * v_scale[:, None]).to(q.dtype)
acc_o += tl.dot(p.to(v.dtype), v)
m_i = m_ij
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
@@ -205,6 +234,8 @@ def _gqa_sparse_fwd_kernel(
def _gqa_sparse_decode_kernel(
q_ptr, # [total_q, num_heads, head_dim]
kv_cache_ptr, # main cache: [num_blocks, num_kv_heads, 128, 2*head_dim]
k_scale_ptr,
v_scale_ptr,
t_ptr, # topk_idx: [num_kv_heads, total_q, topk]
o_ptr, # partial out: [NUM_TOPK_CHUNKS, total_q, num_heads, head_dim]
lse_ptr, # partial lse (log2): [NUM_TOPK_CHUNKS, total_q, num_heads]
@@ -223,6 +254,10 @@ def _gqa_sparse_decode_kernel(
stride_kv_h,
stride_kv_pos,
stride_kv_d,
stride_ks_h,
stride_ks_t,
stride_vs_h,
stride_vs_t,
stride_th,
stride_tn,
stride_tk,
@@ -239,6 +274,7 @@ def _gqa_sparse_decode_kernel(
BLOCK_SIZE_H: tl.constexpr,
BLOCK_SIZE_D: tl.constexpr,
USE_FP8: tl.constexpr, # fp8 KV cache: dequantize K/V to q.dtype on load
KV_SCALE_MODE: tl.constexpr, # 0: none, 1: scalar, 2: [kv_head, token]
USE_PDL: tl.constexpr,
):
sm_scale_log2e = sm_scale * 1.4426950409
@@ -305,6 +341,17 @@ def _gqa_sparse_decode_kernel(
)
if USE_FP8:
k = k.to(q.dtype)
if KV_SCALE_MODE == 1:
k = (k * tl.load(k_scale_ptr)).to(q.dtype)
elif KV_SCALE_MODE == 2:
k_scale = tl.load(
k_scale_ptr
+ pid_kh * stride_ks_h
+ (page * BLOCK_SIZE_K + off_n) * stride_ks_t,
mask=pos_mask,
other=1.0,
)
k = (k * k_scale[None, :]).to(q.dtype)
qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
qk += tl.where(pos_mask[None, :], 0, float("-inf"))
qk += tl.dot(q, k) * sm_scale_log2e
@@ -323,6 +370,17 @@ def _gqa_sparse_decode_kernel(
)
if USE_FP8:
v = v.to(q.dtype)
if KV_SCALE_MODE == 1:
v = (v * tl.load(v_scale_ptr)).to(q.dtype)
elif KV_SCALE_MODE == 2:
v_scale = tl.load(
v_scale_ptr
+ pid_kh * stride_vs_h
+ (page * BLOCK_SIZE_K + off_n) * stride_vs_t,
mask=pos_mask,
other=1.0,
)
v = (v * v_scale[:, None]).to(q.dtype)
acc_o += tl.dot(p.to(v.dtype), v)
m_i = m_ij
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
@@ -410,6 +468,48 @@ def _merge_topk_attn_out_kernel(
# ---------------------------------------------------------------------------
# Python wrappers
# ---------------------------------------------------------------------------
_KV_SCALE_NONE = 0
_KV_SCALE_SCALAR = 1
_KV_SCALE_PER_TOKEN_HEAD = 2
def _kv_scale_args(
output: torch.Tensor,
num_kv_heads: int,
k_scale: torch.Tensor | None,
v_scale: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, int, int, int, int, int]:
if k_scale is None and v_scale is None:
return output, output, 0, 0, 0, 0, _KV_SCALE_NONE
if k_scale is None or v_scale is None:
raise ValueError("k_scale and v_scale must be both provided or both None")
if k_scale.device != output.device or v_scale.device != output.device:
raise ValueError("k_scale and v_scale must be on the same device as output")
if k_scale.numel() == 1 and v_scale.numel() == 1:
return k_scale, v_scale, 0, 0, 0, 0, _KV_SCALE_SCALAR
if k_scale.dim() == 2 and v_scale.dim() == 2:
if k_scale.shape[0] != num_kv_heads or v_scale.shape[0] != num_kv_heads:
raise ValueError(
"per-token/head KV scales must have shape "
f"[{num_kv_heads}, max_kv_tokens]"
)
if k_scale.shape != v_scale.shape:
raise ValueError("k_scale and v_scale must have matching shapes")
return (
k_scale,
v_scale,
k_scale.stride(0),
k_scale.stride(1),
v_scale.stride(0),
v_scale.stride(1),
_KV_SCALE_PER_TOKEN_HEAD,
)
raise ValueError(
"MiniMax-M3 sparse attention supports scalar KV scales or "
"[num_kv_heads, max_kv_tokens] per-token/head scales"
)
@torch.no_grad()
def minimax_m3_sparse_attn(
q: torch.Tensor, # [total_q, num_heads, head_dim]
@@ -423,6 +523,8 @@ def minimax_m3_sparse_attn(
num_kv_heads: int,
sm_scale: float,
output: torch.Tensor, # [total_q, num_heads, head_dim]
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
) -> None:
"""GQA block-sparse attention over the selected blocks. block_size_q == 1."""
total_q, num_heads, head_dim = q.shape
@@ -430,10 +532,33 @@ def minimax_m3_sparse_attn(
topk = topk_idx.shape[-1]
gqa_group_size = num_heads // num_kv_heads
use_fp8 = kv_cache.dtype in _FP8_DTYPES
(
k_scale_arg,
v_scale_arg,
stride_ks_h,
stride_ks_t,
stride_vs_h,
stride_vs_t,
kv_scale_mode,
) = (
_kv_scale_args(output, num_kv_heads, k_scale, v_scale)
if use_fp8
else (
output,
output,
0,
0,
0,
0,
_KV_SCALE_NONE,
)
)
grid = (max_query_len, num_kv_heads, batch)
_gqa_sparse_fwd_kernel[grid](
q,
kv_cache,
k_scale_arg,
v_scale_arg,
topk_idx,
output,
block_table,
@@ -454,6 +579,10 @@ def minimax_m3_sparse_attn(
kv_cache.stride(1),
kv_cache.stride(2),
kv_cache.stride(3),
stride_ks_h,
stride_ks_t,
stride_vs_h,
stride_vs_t,
topk_idx.stride(0),
topk_idx.stride(1),
topk_idx.stride(2),
@@ -464,6 +593,7 @@ def minimax_m3_sparse_attn(
BLOCK_SIZE_Q=1,
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
USE_FP8=use_fp8,
KV_SCALE_MODE=kv_scale_mode,
)
@@ -478,6 +608,8 @@ def minimax_m3_sparse_attn_decode(
sm_scale: float,
output: torch.Tensor, # [total_q, num_heads, head_dim]
decode_query_len: int,
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
) -> None:
"""GQA block-sparse attention for decode (split-K over the top-k blocks)."""
total_q, num_heads, head_dim = q.shape
@@ -485,6 +617,27 @@ def minimax_m3_sparse_attn_decode(
max_topk = topk_idx.shape[-1]
gqa_group_size = num_heads // num_kv_heads
use_fp8 = kv_cache.dtype in _FP8_DTYPES
(
k_scale_arg,
v_scale_arg,
stride_ks_h,
stride_ks_t,
stride_vs_h,
stride_vs_t,
kv_scale_mode,
) = (
_kv_scale_args(output, num_kv_heads, k_scale, v_scale)
if use_fp8
else (
output,
output,
0,
0,
0,
0,
_KV_SCALE_NONE,
)
)
use_pdl = current_platform.is_arch_support_pdl()
# `launch_pdl` is a Triton runtime kwarg only some backends accept (CUDA
# SM9+); this ROCm Triton rejects it even when False ("Keyword argument
@@ -505,6 +658,8 @@ def minimax_m3_sparse_attn_decode(
_gqa_sparse_decode_kernel[grid](
q,
kv_cache,
k_scale_arg,
v_scale_arg,
topk_idx,
o_partial,
lse_partial,
@@ -523,6 +678,10 @@ def minimax_m3_sparse_attn_decode(
kv_cache.stride(1),
kv_cache.stride(2),
kv_cache.stride(3),
stride_ks_h,
stride_ks_t,
stride_vs_h,
stride_vs_t,
topk_idx.stride(0),
topk_idx.stride(1),
topk_idx.stride(2),
@@ -537,6 +696,7 @@ def minimax_m3_sparse_attn_decode(
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
NUM_TOPK_CHUNKS=num_topk_chunks,
USE_FP8=use_fp8,
KV_SCALE_MODE=kv_scale_mode,
USE_PDL=use_pdl,
**pdl_launch,
)
@@ -20,6 +20,7 @@ from typing import ClassVar
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context
@@ -58,6 +59,21 @@ from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
logger = init_logger(__name__)
def _minimax_m3_aiter_sparse_pa_requested() -> bool:
return rocm_aiter_ops.is_enabled() and rocm_aiter_ops.is_shuffle_kv_cache_enabled()
def minimax_m3_use_aiter_sparse_pa(num_kv_heads: int) -> bool:
"""Whether to use the ROCm AITER page-16 sparse PA prototype."""
requested = _minimax_m3_aiter_sparse_pa_requested()
if requested and num_kv_heads != 1:
raise ValueError(
"MiniMax M3 AITER sparse paged attention requires "
f"num_kv_heads == 1 per tensor-parallel rank, got {num_kv_heads}."
)
return requested
class MiniMaxM3SparseBackend(AttentionBackend):
"""Block-sparse GQA backend for MiniMax M3 sparse attention layers."""
@@ -104,6 +120,12 @@ class MiniMaxM3SparseBackend(AttentionBackend):
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
if minimax_m3_use_aiter_sparse_pa(num_kv_heads):
# AITER's assembly paged-attention kernels require independently
# contiguous K and V storage. Keep that specialized layout behind
# the shuffle flag while every other implementation uses the
# packed-content contract introduced by #44455.
return (num_blocks, 2, block_size, num_kv_heads, head_size)
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
@@ -116,6 +138,11 @@ class MiniMaxM3SparseBackend(AttentionBackend):
# layout we want.
if include_num_layers_dimension:
raise NotImplementedError # no cross-layer KV blocks in M3
if _minimax_m3_aiter_sparse_pa_requested():
# The AITER page-16 sparse PA path reinterprets the K and V slices
# as separate SHUFFLE caches. Keep K/V physically separated so
# kv_cache[:, 0] and kv_cache[:, 1] are contiguous byte ranges.
return (1, 0, 2, 3, 4)
cache_layout = get_kv_cache_layout()
if cache_layout == "NHD":
# (num_blocks, block_size, num_kv_heads, 2*head_size)
@@ -365,6 +392,8 @@ class MiniMaxM3SparseTritonImpl(MiniMaxM3SparseImpl):
kv_cache = (
kv_cache.view(self.kv_cache_fp8_dtype) if self.use_fp8_kv else kv_cache
)
k_scale = getattr(layer, "_k_scale", None) if self.use_fp8_kv else None
v_scale = getattr(layer, "_v_scale", None) if self.use_fp8_kv else None
# Decode [:nd]: split-K over the selected blocks (request-major chunks).
if main_md.num_decodes > 0:
@@ -380,6 +409,8 @@ class MiniMaxM3SparseTritonImpl(MiniMaxM3SparseImpl):
self.scale,
out[:nd],
d.decode_query_len,
k_scale=k_scale,
v_scale=v_scale,
)
# Prefill [nd:]: cu_seqlens_q already rebased to 0.
@@ -398,6 +429,8 @@ class MiniMaxM3SparseTritonImpl(MiniMaxM3SparseImpl):
self.num_kv_heads,
self.scale,
out[nd:],
k_scale=k_scale,
v_scale=v_scale,
)
return output
@@ -406,27 +439,38 @@ def select_main_impl_cls(
*,
topk_blocks: int,
kv_cache_dtype: str,
num_kv_heads: int,
) -> type[MiniMaxM3SparseImpl]:
"""Pick the main attend impl off the main KV-cache dtype.
Blackwell (SM100) uses the MSA attend for supported top-k block counts
when the KV cache is BF16 or FP8 E4M3; non-Blackwell and FP8 E5M2 fall
back to Triton. The MSA module is imported lazily so AMD/non-SM100 never
import fmha_sm100.
when the KV cache is BF16 or FP8 E4M3; MI355 uses AITER sparse PA
with shuffle KV cache layout; Other platforms and FP8 E5M2 fall
back to Triton. The MSA modules are imported lazily avoid import errors
on unsupported platforms.
"""
use_aiter_sparse_pa = minimax_m3_use_aiter_sparse_pa(num_kv_heads)
use_msa = (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
and topk_blocks in (4, 8, 16, 32)
and kv_cache_dtype != "fp8_e5m2"
)
selected = "MSA" if use_msa else "Triton"
selected = (
"AITER_SPARSE_PA" if use_aiter_sparse_pa else ("MSA" if use_msa else "Triton")
)
logger.info_once(
"MiniMax M3 sparse attention selected %s (kv_cache_dtype=%s, topk_blocks=%s)",
selected,
kv_cache_dtype,
topk_blocks,
)
if use_aiter_sparse_pa:
from vllm.models.minimax_m3.amd.sparse_attention_msa import (
MiniMaxM3SparseAiterPAImpl,
)
return MiniMaxM3SparseAiterPAImpl
if use_msa:
from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
MiniMaxM3SparseMSAImpl,
+7
View File
@@ -25,6 +25,7 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.activation import SiluAndMulWithClamp
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention.attention import set_default_quant_scales
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
fused_allreduce_gemma_rms_norm,
@@ -491,6 +492,11 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
self.kv_cache_dtype, vllm_config.model_config
)
# MiniMax-M3 sparse attention owns its KV-cache insert/read path instead
# of wrapping the generic Attention module. Keep the same runtime scale
# attributes so FP8 KV reads can honor vLLM's per-layer descale contract.
self.calculate_kv_scales = False
set_default_quant_scales(self, register_buffer=True)
# Indexer side-cache dtype, mirroring --kv-cache-dtype for the main
# cache (--attention-config '{"indexer_kv_dtype": ...}').
self.indexer_kv_dtype = vllm_config.attention_config.indexer_kv_dtype
@@ -506,6 +512,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
self.impl: MiniMaxM3SparseImpl = select_main_impl_cls( # type: ignore[assignment]
topk_blocks=sparse_cfg["sparse_topk_blocks"],
kv_cache_dtype=self.kv_cache_dtype,
num_kv_heads=self.num_kv_heads,
)(
self.num_heads,
self.head_dim,
@@ -49,6 +49,8 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
kv_cache = (
kv_cache.view(self.kv_cache_fp8_dtype) if self.use_fp8_kv else kv_cache
)
k_scale = getattr(layer, "_k_scale", None) if self.use_fp8_kv else None
v_scale = getattr(layer, "_v_scale", None) if self.use_fp8_kv else None
# Decode [:nd]: Triton split-K placeholder (no MSA decode yet).
if main_md.num_decodes > 0:
@@ -64,6 +66,8 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
self.scale,
out[:nd],
d.decode_query_len,
k_scale=k_scale,
v_scale=v_scale,
)
# Prefill [nd:]: MSA sparse FMHA over the selected blocks.