Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Claude <noreply@anthropic.com>
606 lines
23 KiB
Python
606 lines
23 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Main block-sparse GQA attention for MiniMax M3 sparse layers.
|
|
|
|
The lightning indexer (``indexer.py``) selects the top-k KV blocks (written into
|
|
the shared ``layer.topk_indices_buffer``); this module holds the main attention
|
|
that attends only to those blocks: the paged K/V cache backend, its metadata +
|
|
builder, and the impl that reads the indexer's top-k from that buffer. The Triton
|
|
attend kernel lives here; the SM100 (MSA)
|
|
``build_k2q_csr`` + ``sparse_atten_func`` attend lives in
|
|
``nvidia/sparse_attention_msa.py``.
|
|
|
|
``MiniMaxM3SparseBackend`` and ``MiniMaxM3SparseMetadata`` are referenced by the
|
|
attention-backend registry (by dotted path) and by spec-decode, so they must
|
|
keep these names and stay in this module.
|
|
"""
|
|
|
|
from dataclasses import dataclass
|
|
from typing import ClassVar
|
|
|
|
import torch
|
|
|
|
from vllm._aiter_ops import rocm_aiter_ops
|
|
from vllm.config import VllmConfig
|
|
from vllm.config.cache import CacheDType
|
|
from vllm.forward_context import get_forward_context
|
|
from vllm.logger import init_logger
|
|
from vllm.models.minimax_m3.common.ops.sparse_attn import SPARSE_BLOCK_SIZE
|
|
from vllm.platforms import current_platform
|
|
|
|
# AMD/ROCm uses the gfx942/gfx950-optimized block-sparse kernels in amd.ops;
|
|
# every other platform uses the generic common.ops implementation.
|
|
if current_platform.is_rocm():
|
|
from vllm.models.minimax_m3.amd.ops.sparse_attn import (
|
|
minimax_m3_sparse_attn,
|
|
minimax_m3_sparse_attn_decode,
|
|
)
|
|
else:
|
|
from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
|
minimax_m3_sparse_attn,
|
|
minimax_m3_sparse_attn_decode,
|
|
)
|
|
from vllm.v1.attention.backend import (
|
|
AttentionBackend,
|
|
AttentionCGSupport,
|
|
AttentionImplBase,
|
|
AttentionLayer,
|
|
AttentionMetadata,
|
|
AttentionMetadataBuilder,
|
|
CommonAttentionMetadata,
|
|
MultipleOf,
|
|
)
|
|
from vllm.v1.attention.backends.utils import (
|
|
split_decodes_and_prefills,
|
|
)
|
|
from vllm.v1.kv_cache_interface import (
|
|
AttentionSpec,
|
|
KVCacheLayout,
|
|
get_kv_quant_mode,
|
|
is_quantized_kv_cache,
|
|
)
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
def _minimax_m3_aiter_sparse_pa_requested() -> bool:
|
|
return rocm_aiter_ops.is_enabled() and rocm_aiter_ops.is_shuffle_kv_cache_enabled()
|
|
|
|
|
|
def minimax_m3_rebase_slots_to_page16(
|
|
slot_mapping: torch.Tensor,
|
|
block_size: int,
|
|
out: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
"""Rebase a token slot mapping onto AITER's page-16 page numbering.
|
|
|
|
AITER's KV writer derives the destination page from ``slot // 16`` and
|
|
assumes pages are numbered consecutively. When the resolved layout stores
|
|
both K/V sides inside a block, a block spans twice as many pages, so only
|
|
the block component of the slot doubles -- the offset within the page must
|
|
not move. Clamping before the division leaves padding slots at their
|
|
negative sentinel, which is what tells the writer to skip them.
|
|
"""
|
|
if out is None:
|
|
out = torch.empty_like(slot_mapping)
|
|
torch.clamp(slot_mapping, min=0, out=out)
|
|
return (
|
|
out.div_(block_size, rounding_mode="floor").mul_(block_size).add_(slot_mapping)
|
|
)
|
|
|
|
|
|
def minimax_m3_query_token_positions(
|
|
cu_seqlens_q: torch.Tensor,
|
|
prefix_lens: torch.Tensor,
|
|
total_q: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Map each prefill query token to its request id and absolute KV position.
|
|
|
|
Both tensors are the same for every sparse layer in a step, so the AITER
|
|
sparse PA block-table builder reads them from the metadata instead of
|
|
rebuilding them once per layer.
|
|
"""
|
|
pos = torch.arange(total_q, dtype=torch.int32, device=cu_seqlens_q.device)
|
|
query_req_id = torch.searchsorted(
|
|
cu_seqlens_q[1:].contiguous(), pos, right=True
|
|
).to(torch.int32)
|
|
query_abs_pos = (prefix_lens[query_req_id] + (pos - cu_seqlens_q[query_req_id])).to(
|
|
torch.int32
|
|
)
|
|
return query_req_id, query_abs_pos
|
|
|
|
|
|
def minimax_m3_use_aiter_sparse_pa(
|
|
num_kv_heads: int, *, emits_sparse_block_table: bool = False
|
|
) -> bool:
|
|
"""Whether to use the ROCm AITER page-16 sparse PA prototype.
|
|
|
|
More than one KV head per rank is only served when the layer's indexer
|
|
emits the attend's page table itself, since the Triton builders this path
|
|
otherwise falls back to address one head's cache. Whether an indexer does
|
|
that is a ROCm-side fact, so its caller passes it in.
|
|
"""
|
|
requested = _minimax_m3_aiter_sparse_pa_requested()
|
|
if requested and num_kv_heads != 1 and not emits_sparse_block_table:
|
|
raise ValueError(
|
|
"MiniMax M3 AITER sparse paged attention requires "
|
|
f"num_kv_heads == 1 per tensor-parallel rank, got {num_kv_heads}, "
|
|
"unless the AITER indexer emits the attend's page table. It is not "
|
|
"in use here; the 'MiniMax M3 indexer: AITER unavailable' log line "
|
|
"says why."
|
|
)
|
|
return requested
|
|
|
|
|
|
class MiniMaxM3SparseBackend(AttentionBackend):
|
|
"""Block-sparse GQA backend for MiniMax M3 sparse attention layers."""
|
|
|
|
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16, torch.float16]
|
|
# bf16 or fp8 (e4m3/e5m2): the Triton kernels dequant fp8 before the dots.
|
|
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
|
"bfloat16",
|
|
"fp8",
|
|
"fp8_e4m3",
|
|
"fp8_e5m2",
|
|
]
|
|
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "MINIMAX_M3_SPARSE"
|
|
|
|
@staticmethod
|
|
def get_impl_cls() -> type["MiniMaxM3SparseImpl"]:
|
|
# Concrete impl chosen by select_main_impl_cls; base for introspection.
|
|
return MiniMaxM3SparseImpl
|
|
|
|
@staticmethod
|
|
def get_builder_cls() -> type["MiniMaxM3SparseMetadataBuilder"]:
|
|
return MiniMaxM3SparseMetadataBuilder
|
|
|
|
@classmethod
|
|
def get_supported_head_sizes(cls) -> list[int]:
|
|
return [128]
|
|
|
|
@staticmethod
|
|
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
|
# Page size == sparse block size (one sparse block per KV page).
|
|
return [128]
|
|
|
|
@classmethod
|
|
def is_sparse(cls) -> bool:
|
|
return True
|
|
|
|
@classmethod
|
|
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...] | None:
|
|
# AITER sparse PA reads K and V as separate page-16 tensors. LHBNC
|
|
# gives each side a dense plane; LBHNC interleaves the sides inside a
|
|
# block, which the page-id rebasing in ops/sparse_pa.py absorbs. LBHNC
|
|
# is listed first because these layers publish two head slots while the
|
|
# indexer publishes one, and mixed HNC shapes need a block-compact
|
|
# layout to share the block-id pool.
|
|
if _minimax_m3_aiter_sparse_pa_requested():
|
|
return (KVCacheLayout.LBHNC, KVCacheLayout.LHBNC)
|
|
return super().supported_kv_cache_layouts()
|
|
|
|
|
|
@dataclass
|
|
class MiniMaxM3SparsePrefillMetadata:
|
|
"""Per-prefill state; ``cu_seqlens_k``/``total_kv_blocks`` feed the MSA CSR."""
|
|
|
|
cu_seqlens_q: torch.Tensor # [num_prefills + 1] int32, rebased to 0
|
|
cu_seqlens_k: torch.Tensor # [num_prefills + 1] int32, cumulative KV lengths
|
|
seq_lens: torch.Tensor # [num_prefills] int32, total KV lengths
|
|
context_lens: torch.Tensor # [num_prefills] int32 (cached/context tokens)
|
|
block_table: torch.Tensor
|
|
max_query_len: int
|
|
max_seq_len: int
|
|
total_kv_blocks: int
|
|
|
|
# Per-query-token request id and absolute KV position, built once per step
|
|
# for the AITER sparse PA block-table kernel. See
|
|
# ``minimax_m3_query_token_positions``.
|
|
query_req_id: torch.Tensor | None = None
|
|
query_abs_pos: torch.Tensor | None = None
|
|
|
|
|
|
@dataclass
|
|
class MiniMaxM3SparseDecodeMetadata:
|
|
"""Per-decode state (cudagraph-safe). ``decode_query_len`` is the uniform
|
|
per-request query length (1, or 1 + num_speculative_tokens)."""
|
|
|
|
seq_lens: torch.Tensor # [num_decodes] int32
|
|
block_table: torch.Tensor
|
|
decode_query_len: int
|
|
|
|
|
|
@dataclass
|
|
class MiniMaxM3SparseMetadata(AttentionMetadata):
|
|
"""Sparse-attention metadata, split into prefill and decode sub-metadata."""
|
|
|
|
seq_lens: torch.Tensor
|
|
max_seq_len: int
|
|
slot_mapping: torch.Tensor
|
|
|
|
num_actual_tokens: int # total query tokens (decode-first batch)
|
|
|
|
# Split counts (batch reordered decode-first).
|
|
num_decodes: int
|
|
num_decode_tokens: int
|
|
num_prefills: int
|
|
num_prefill_tokens: int
|
|
|
|
prefill: MiniMaxM3SparsePrefillMetadata | None = None
|
|
decode: MiniMaxM3SparseDecodeMetadata | None = None
|
|
|
|
# ``slot_mapping`` rebased onto the page-16 numbering AITER's KV writer
|
|
# uses when both K/V sides share a block. Only set for the AITER sparse PA
|
|
# path; see ``MiniMaxM3SparseMetadataBuilder.build``.
|
|
page16_slot_mapping: torch.Tensor | None = None
|
|
|
|
|
|
class MiniMaxM3SparseMetadataBuilder(AttentionMetadataBuilder[MiniMaxM3SparseMetadata]):
|
|
# Full cudagraphs for uniform decode batches, incl. spec-decode verify
|
|
# batches with >1 query token/request.
|
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
|
# Raised to 1 + num_speculative_tokens by _init_reorder_batch_threshold when
|
|
# spec decode is on; must match the indexer builder so the splits agree.
|
|
reorder_batch_threshold: int = 1
|
|
|
|
def __init__(
|
|
self,
|
|
kv_cache_spec: AttentionSpec,
|
|
layer_names: list[str],
|
|
vllm_config: VllmConfig,
|
|
device: torch.device,
|
|
) -> None:
|
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
|
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
|
# Stable context-length buffer for decode cudagraph replays.
|
|
self.context_len_buffer = torch.empty(
|
|
vllm_config.scheduler_config.max_num_batched_tokens,
|
|
dtype=torch.int32,
|
|
device=device,
|
|
)
|
|
# Every sparse layer shares one slot mapping, so the AITER page-16
|
|
# rebase is done here once per step instead of once per layer. Stable
|
|
# buffer for the same reason as the context lengths above.
|
|
# The request itself, not the gate: the buffer is the same whatever the
|
|
# head count, and a builder cannot see which indexer its layers picked.
|
|
self.use_aiter_sparse_pa = _minimax_m3_aiter_sparse_pa_requested()
|
|
self.page16_slot_mapping_buffer: torch.Tensor | None = None
|
|
if self.use_aiter_sparse_pa:
|
|
self.page16_slot_mapping_buffer = torch.empty(
|
|
vllm_config.scheduler_config.max_num_batched_tokens,
|
|
dtype=torch.int64,
|
|
device=device,
|
|
)
|
|
|
|
def build(
|
|
self,
|
|
common_prefix_len: int,
|
|
common_attn_metadata: CommonAttentionMetadata,
|
|
fast_build: bool = False,
|
|
) -> MiniMaxM3SparseMetadata:
|
|
num_reqs = common_attn_metadata.num_reqs
|
|
num_tokens = common_attn_metadata.num_actual_tokens
|
|
query_start_loc = common_attn_metadata.query_start_loc
|
|
seq_lens = common_attn_metadata.seq_lens
|
|
block_table = common_attn_metadata.block_table_tensor
|
|
|
|
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
|
split_decodes_and_prefills(
|
|
common_attn_metadata,
|
|
decode_threshold=self.reorder_batch_threshold,
|
|
require_uniform=True,
|
|
)
|
|
)
|
|
assert num_decodes + num_prefills == num_reqs
|
|
assert num_decode_tokens + num_prefill_tokens == num_tokens
|
|
|
|
# Decode-first batch: context lengths into the stable cudagraph buffer.
|
|
context_lens = self.context_len_buffer[:num_reqs]
|
|
context_lens.copy_(
|
|
common_attn_metadata.compute_num_computed_tokens(), non_blocking=True
|
|
)
|
|
|
|
prefill_metadata: MiniMaxM3SparsePrefillMetadata | None = None
|
|
if num_prefills > 0:
|
|
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
|
assert seq_lens_cpu is not None
|
|
prefill_seq_lens_cpu = seq_lens_cpu[num_decodes:]
|
|
prefill_total_kv_blocks = (
|
|
((prefill_seq_lens_cpu + SPARSE_BLOCK_SIZE - 1) // SPARSE_BLOCK_SIZE)
|
|
.sum()
|
|
.item()
|
|
)
|
|
prefill_kv_lens = seq_lens[num_decodes:]
|
|
prefill_cu_seqlens_k = torch.empty(
|
|
num_prefills + 1, dtype=torch.int32, device=seq_lens.device
|
|
)
|
|
if current_platform.is_rocm():
|
|
prefill_cu_seqlens_k[:1].zero_()
|
|
else:
|
|
prefill_cu_seqlens_k[0] = 0
|
|
torch.cumsum(prefill_kv_lens, dim=0, out=prefill_cu_seqlens_k[1:])
|
|
prefill_cu_seqlens_q = (
|
|
query_start_loc[num_decodes:] - num_decode_tokens
|
|
).to(torch.int32)
|
|
prefill_context_lens = context_lens[num_decodes:]
|
|
query_req_id = query_abs_pos = None
|
|
if self.use_aiter_sparse_pa:
|
|
query_req_id, query_abs_pos = minimax_m3_query_token_positions(
|
|
prefill_cu_seqlens_q, prefill_context_lens, num_prefill_tokens
|
|
)
|
|
prefill_metadata = MiniMaxM3SparsePrefillMetadata(
|
|
cu_seqlens_q=prefill_cu_seqlens_q,
|
|
cu_seqlens_k=prefill_cu_seqlens_k,
|
|
seq_lens=prefill_kv_lens,
|
|
context_lens=prefill_context_lens,
|
|
block_table=block_table[num_decodes:],
|
|
max_query_len=common_attn_metadata.max_query_len,
|
|
max_seq_len=common_attn_metadata.max_seq_len,
|
|
total_kv_blocks=prefill_total_kv_blocks,
|
|
query_req_id=query_req_id,
|
|
query_abs_pos=query_abs_pos,
|
|
)
|
|
|
|
decode_metadata: MiniMaxM3SparseDecodeMetadata | None = None
|
|
if num_decodes > 0:
|
|
qsl_cpu = common_attn_metadata.query_start_loc_cpu
|
|
query_lens_cpu = qsl_cpu[1 : num_decodes + 1] - qsl_cpu[:num_decodes]
|
|
decode_query_len = int(query_lens_cpu[0].item())
|
|
assert decode_query_len > 0
|
|
assert torch.all(
|
|
(query_lens_cpu == decode_query_len) | (query_lens_cpu == 0)
|
|
)
|
|
assert num_decode_tokens == num_decodes * decode_query_len
|
|
decode_metadata = MiniMaxM3SparseDecodeMetadata(
|
|
seq_lens=seq_lens[:num_decodes],
|
|
block_table=block_table[:num_decodes],
|
|
decode_query_len=decode_query_len,
|
|
)
|
|
|
|
page16_slot_mapping = None
|
|
if self.page16_slot_mapping_buffer is not None:
|
|
slot_mapping = common_attn_metadata.slot_mapping
|
|
page16_slot_mapping = minimax_m3_rebase_slots_to_page16(
|
|
slot_mapping,
|
|
self.kv_cache_spec.block_size,
|
|
out=self.page16_slot_mapping_buffer[: slot_mapping.shape[0]],
|
|
)
|
|
|
|
return MiniMaxM3SparseMetadata(
|
|
seq_lens=seq_lens,
|
|
max_seq_len=common_attn_metadata.max_seq_len,
|
|
slot_mapping=common_attn_metadata.slot_mapping,
|
|
page16_slot_mapping=page16_slot_mapping,
|
|
num_actual_tokens=num_tokens,
|
|
num_decodes=num_decodes,
|
|
num_decode_tokens=num_decode_tokens,
|
|
num_prefills=num_prefills,
|
|
num_prefill_tokens=num_prefill_tokens,
|
|
prefill=prefill_metadata,
|
|
decode=decode_metadata,
|
|
)
|
|
|
|
|
|
class MiniMaxM3SparseImpl(AttentionImplBase[MiniMaxM3SparseMetadata]):
|
|
"""Abstract base for block-sparse GQA over the indexer-selected blocks.
|
|
|
|
Inherits ``AttentionImplBase`` for a custom forward signature (the layer
|
|
pre-inserts K/V and runs the indexer, which writes the selected blocks into
|
|
the shared ``layer.topk_indices_buffer``; the attend reads them back from
|
|
there). The Triton and MSA subclasses each own a full ``forward`` -- no
|
|
shared forward code.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
num_heads: int,
|
|
head_size: int,
|
|
scale: float,
|
|
num_kv_heads: int | None = None,
|
|
kv_cache_dtype: str = "auto",
|
|
*,
|
|
topk_blocks: int,
|
|
sparse_block_size: int,
|
|
msa_decode_backend: str = "triton",
|
|
) -> None:
|
|
self.num_heads = num_heads
|
|
self.head_size = head_size
|
|
self.scale = scale
|
|
self.num_kv_heads = num_kv_heads if num_kv_heads is not None else num_heads
|
|
self.kv_cache_dtype = kv_cache_dtype
|
|
self.use_fp8_kv = is_quantized_kv_cache(kv_cache_dtype)
|
|
if "e5m2" in kv_cache_dtype:
|
|
self.kv_cache_fp8_dtype = (
|
|
torch.float8_e5m2fnuz
|
|
if current_platform.is_fp8_fnuz()
|
|
else torch.float8_e5m2
|
|
)
|
|
else:
|
|
self.kv_cache_fp8_dtype = current_platform.fp8_dtype()
|
|
# Sparse selection parameters (block_size == page size == SPARSE_BLOCK_SIZE).
|
|
self.topk_blocks = topk_blocks
|
|
self.block_size = sparse_block_size
|
|
|
|
def forward(
|
|
self,
|
|
layer: AttentionLayer,
|
|
query: torch.Tensor,
|
|
kv_cache: torch.Tensor,
|
|
output: torch.Tensor,
|
|
*,
|
|
query_fp8: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
"""Attend the queries to the indexer-selected blocks. Per kernel.
|
|
|
|
The indexer has already written the top-k block ids into
|
|
``layer.topk_indices_buffer`` (decode at ``[:, :nd]``, prefill at
|
|
``[:, nd:num_tokens]``); the attend reads them from there.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
def should_use_msa_decode(self, layer_name: str) -> bool:
|
|
return False
|
|
|
|
|
|
class MiniMaxM3SparseTritonImpl(MiniMaxM3SparseImpl):
|
|
"""Triton block-sparse attend (``minimax_m3_sparse_attn``) + Triton decode."""
|
|
|
|
def forward(
|
|
self,
|
|
layer: AttentionLayer,
|
|
query: torch.Tensor,
|
|
kv_cache: torch.Tensor,
|
|
output: torch.Tensor,
|
|
*,
|
|
query_fp8: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
attn_metadata = get_forward_context().attn_metadata
|
|
if not isinstance(attn_metadata, dict):
|
|
return output # profiling run; caches unbound
|
|
main_md = attn_metadata[layer.layer_name] # type: ignore[attr-defined]
|
|
assert isinstance(main_md, MiniMaxM3SparseMetadata)
|
|
|
|
nd = main_md.num_decode_tokens
|
|
num_tokens = main_md.num_actual_tokens
|
|
# Indexer top-k from the shared buffer: decode [:, :nd], prefill [:, nd:].
|
|
topk_buffer = layer.topk_indices_buffer # type: ignore[attr-defined]
|
|
assert topk_buffer is not None
|
|
|
|
topk = (
|
|
topk_buffer
|
|
if current_platform.is_rocm()
|
|
else topk_buffer[:num_tokens].transpose(0, 1)
|
|
)
|
|
assert topk is not None
|
|
hd = self.head_size
|
|
q = query[:num_tokens].view(-1, self.num_heads, hd)
|
|
out = output[:num_tokens].view(-1, self.num_heads, hd)
|
|
kv_cache = (
|
|
kv_cache.view(self.kv_cache_fp8_dtype) if self.use_fp8_kv else kv_cache
|
|
)
|
|
k_scale = getattr(layer, "_k_scale", None) if self.use_fp8_kv else None
|
|
v_scale = getattr(layer, "_v_scale", None) if self.use_fp8_kv else None
|
|
|
|
# Decode [:nd]: split-K over the selected blocks (request-major chunks).
|
|
if main_md.num_decodes > 0:
|
|
d = main_md.decode
|
|
assert d is not None
|
|
minimax_m3_sparse_attn_decode(
|
|
q[:nd],
|
|
kv_cache,
|
|
topk[:, :nd, :],
|
|
d.block_table,
|
|
d.seq_lens,
|
|
self.num_kv_heads,
|
|
self.scale,
|
|
out[:nd],
|
|
d.decode_query_len,
|
|
k_scale=k_scale,
|
|
v_scale=v_scale,
|
|
)
|
|
|
|
# Prefill [nd:]: cu_seqlens_q already rebased to 0.
|
|
if main_md.num_prefills > 0:
|
|
p = main_md.prefill
|
|
assert p is not None
|
|
minimax_m3_sparse_attn(
|
|
q[nd:],
|
|
kv_cache,
|
|
topk[:, nd:num_tokens, :],
|
|
p.block_table,
|
|
p.cu_seqlens_q,
|
|
p.seq_lens,
|
|
p.context_lens,
|
|
p.max_query_len,
|
|
self.num_kv_heads,
|
|
self.scale,
|
|
out[nd:],
|
|
k_scale=k_scale,
|
|
v_scale=v_scale,
|
|
)
|
|
return output
|
|
|
|
|
|
def select_main_backend_and_impl_cls(
|
|
*,
|
|
topk_blocks: int,
|
|
kv_cache_dtype: str,
|
|
num_kv_heads: int,
|
|
emits_sparse_block_table: bool = False,
|
|
) -> tuple[type[MiniMaxM3SparseBackend], type[MiniMaxM3SparseImpl]]:
|
|
"""Pick the main attention backend and implementation.
|
|
|
|
Blackwell (SM100) uses the MSA attend for supported top-k block counts
|
|
when the KV cache is BF16, FP8 E4M3 or NVFP4; MI355 uses AITER sparse PA
|
|
with shuffle KV cache layout; Other platforms and FP8 E5M2 fall
|
|
back to Triton. NVFP4 is MSA-only. The MSA modules are imported lazily to
|
|
avoid import errors on unsupported platforms.
|
|
"""
|
|
use_aiter_sparse_pa = minimax_m3_use_aiter_sparse_pa(
|
|
num_kv_heads, emits_sparse_block_table=emits_sparse_block_table
|
|
)
|
|
use_msa = (
|
|
current_platform.is_cuda()
|
|
and current_platform.is_device_capability_family(100)
|
|
and topk_blocks in (4, 8, 16, 32)
|
|
and kv_cache_dtype != "fp8_e5m2"
|
|
)
|
|
use_nvfp4 = get_kv_quant_mode(kv_cache_dtype).is_nvfp4
|
|
if use_nvfp4 and kv_cache_dtype != "nvfp4":
|
|
raise ValueError(
|
|
"MiniMax M3 sparse attention supports only the 'nvfp4' NVFP4 KV "
|
|
f"cache dtype, got {kv_cache_dtype}"
|
|
)
|
|
if use_nvfp4 and (use_aiter_sparse_pa or not use_msa):
|
|
raise ValueError(
|
|
"MiniMax M3 sparse attention supports an NVFP4 KV cache only with "
|
|
f"the SM100 MSA backend (kv_cache_dtype={kv_cache_dtype}, "
|
|
f"topk_blocks={topk_blocks})"
|
|
)
|
|
selected = (
|
|
"AITER_SPARSE_PA" if use_aiter_sparse_pa else ("MSA" if use_msa else "Triton")
|
|
)
|
|
logger.info_once(
|
|
"MiniMax M3 sparse attention selected %s (kv_cache_dtype=%s, topk_blocks=%s)",
|
|
selected,
|
|
kv_cache_dtype,
|
|
topk_blocks,
|
|
)
|
|
if use_aiter_sparse_pa:
|
|
from vllm.models.minimax_m3.amd.sparse_attention_msa import (
|
|
MiniMaxM3SparseAiterPABackend,
|
|
MiniMaxM3SparseAiterPAImpl,
|
|
)
|
|
|
|
return MiniMaxM3SparseAiterPABackend, MiniMaxM3SparseAiterPAImpl
|
|
if use_msa:
|
|
from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
|
|
MiniMaxM3SparseMSABackend,
|
|
MiniMaxM3SparseMSAImpl,
|
|
MiniMaxM3SparseMSANvfp4Backend,
|
|
)
|
|
|
|
if use_nvfp4:
|
|
return MiniMaxM3SparseMSANvfp4Backend, MiniMaxM3SparseMSAImpl
|
|
return MiniMaxM3SparseMSABackend, MiniMaxM3SparseMSAImpl
|
|
return MiniMaxM3SparseBackend, MiniMaxM3SparseTritonImpl
|
|
|
|
|
|
def select_main_impl_cls(
|
|
*,
|
|
topk_blocks: int,
|
|
kv_cache_dtype: str,
|
|
num_kv_heads: int,
|
|
emits_sparse_block_table: bool = False,
|
|
) -> type[MiniMaxM3SparseImpl]:
|
|
"""Backward-compatible implementation-only selector."""
|
|
return select_main_backend_and_impl_cls(
|
|
topk_blocks=topk_blocks,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
num_kv_heads=num_kv_heads,
|
|
emits_sparse_block_table=emits_sparse_block_table,
|
|
)[1]
|