[Attention][MiniMax-M3] Add MSA speculative decode verification (#50032)

This commit is contained in:
Jason Li
2026-08-02 13:46:55 -07:00
committed by GitHub
parent c666810676
commit 0055b8bfa3
15 changed files with 1275 additions and 59 deletions
+7
View File
@@ -2655,6 +2655,8 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
index_q_out: torch.Tensor | None = None,
kv_cache_dtype: str = "auto",
skip_index_branch: bool = False,
q_fp8_out: torch.Tensor | None = None,
q_fp8_scale: float = 1.0,
) -> None:
"""Fused MiniMax-M3 attention pre-processing (in-place).
@@ -2677,6 +2679,9 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
callers skip a separate ``.contiguous()`` copy before the SM100 sparse
attention's flat TMA descriptor.
If ``q_fp8_out`` is given, the same normalized q is also written in FP8
E4M3 using ``q_fp8_scale`` as its dequantization scale.
When ``skip_index_branch`` is true, sparse rows still keep their packed
``[index_q | index_k]`` tail, but the kernel only processes the main q/k/v
branches and main KV cache. This is used by MiniMax-M3 index-topk reuse
@@ -2704,6 +2709,8 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
index_q_out,
kv_cache_dtype,
skip_index_branch,
q_fp8_out,
q_fp8_scale,
)
+15
View File
@@ -11,6 +11,7 @@ from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnu
from vllm.v1.attention.backends.registry import AttentionBackendEnum
IndexerKVDType = Literal["bf16", "fp8", "mxfp4", "nvfp4"]
MiniMaxM3MSADecodeBackend = Literal["triton", "cutlass"]
@config
@@ -20,6 +21,9 @@ class AttentionConfig:
backend: AttentionBackendEnum | None = None
"""Attention backend to use. Use "auto" or None for automatic selection."""
minimax_m3_msa_decode_backend: MiniMaxM3MSADecodeBackend = "triton"
"""Sparse decode kernel used by the MiniMax M3 MSA backend."""
backend_per_kind: dict[str, AttentionBackendEnum] = field(default_factory=dict)
"""Per-KV-cache-group attention backend overrides, keyed by
`KVCacheSpecKind` (e.g. `{"mla_attention": "FLASHINFER_MLA",
@@ -99,6 +103,17 @@ class AttentionConfig:
PyTorch >= 2.9, and 128 for encoder-only attention or older PyTorch
versions."""
def __post_init__(self) -> None:
msa_aliases: dict[AttentionBackendEnum, MiniMaxM3MSADecodeBackend] = {
AttentionBackendEnum.CUTLASS_MSA: "cutlass",
AttentionBackendEnum.TRITON_MSA: "triton",
}
if self.backend in msa_aliases:
self.minimax_m3_msa_decode_backend = msa_aliases[self.backend]
# The alias selects only MiniMax's sparse decode kernel. Dense
# layers still use the platform's normal automatic backend.
self.backend = None
def compute_hash(self) -> str:
"""
Provide a hash that uniquely identifies all the configs
@@ -23,6 +23,8 @@ class MiniMaxM3SparseAiterPAImpl(MiniMaxM3SparseImpl):
query: torch.Tensor,
kv_cache: torch.Tensor,
output: torch.Tensor,
*,
query_fp8: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.models.minimax_m3.amd.ops.sparse_pa import (
minimax_m3_sparse_attn_decode_aiter,
@@ -330,6 +330,7 @@ class MiniMaxM3SparseImpl(AttentionImplBase[MiniMaxM3SparseMetadata]):
*,
topk_blocks: int,
sparse_block_size: int,
msa_decode_backend: str = "triton",
) -> None:
self.num_heads = num_heads
self.head_size = head_size
@@ -355,6 +356,8 @@ class MiniMaxM3SparseImpl(AttentionImplBase[MiniMaxM3SparseMetadata]):
query: torch.Tensor,
kv_cache: torch.Tensor,
output: torch.Tensor,
*,
query_fp8: torch.Tensor | None = None,
) -> torch.Tensor:
"""Attend the queries to the indexer-selected blocks. Per kernel.
@@ -364,6 +367,9 @@ class MiniMaxM3SparseImpl(AttentionImplBase[MiniMaxM3SparseMetadata]):
"""
raise NotImplementedError
def should_use_msa_decode(self, layer_name: str) -> bool:
return False
class MiniMaxM3SparseTritonImpl(MiniMaxM3SparseImpl):
"""Triton block-sparse attend (``minimax_m3_sparse_attn``) + Triton decode."""
@@ -374,6 +380,8 @@ class MiniMaxM3SparseTritonImpl(MiniMaxM3SparseImpl):
query: torch.Tensor,
kv_cache: torch.Tensor,
output: torch.Tensor,
*,
query_fp8: torch.Tensor | None = None,
) -> torch.Tensor:
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
@@ -442,18 +450,18 @@ class MiniMaxM3SparseTritonImpl(MiniMaxM3SparseImpl):
return output
def select_main_impl_cls(
def select_main_backend_and_impl_cls(
*,
topk_blocks: int,
kv_cache_dtype: str,
num_kv_heads: int,
) -> type[MiniMaxM3SparseImpl]:
"""Pick the main attend impl off the main KV-cache dtype.
) -> tuple[type[MiniMaxM3SparseBackend], type[MiniMaxM3SparseImpl]]:
"""Pick the main attention backend and implementation.
Blackwell (SM100) uses the MSA attend for supported top-k block counts
when the KV cache is BF16 or FP8 E4M3; MI355 uses AITER sparse PA
with shuffle KV cache layout; Other platforms and FP8 E5M2 fall
back to Triton. The MSA modules are imported lazily avoid import errors
back to Triton. The MSA modules are imported lazily to avoid import errors
on unsupported platforms.
"""
use_aiter_sparse_pa = minimax_m3_use_aiter_sparse_pa(num_kv_heads)
@@ -477,11 +485,26 @@ def select_main_impl_cls(
MiniMaxM3SparseAiterPAImpl,
)
return MiniMaxM3SparseAiterPAImpl
return MiniMaxM3SparseBackend, MiniMaxM3SparseAiterPAImpl
if use_msa:
from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
MiniMaxM3SparseMSABackend,
MiniMaxM3SparseMSAImpl,
)
return MiniMaxM3SparseMSAImpl
return MiniMaxM3SparseTritonImpl
return MiniMaxM3SparseMSABackend, MiniMaxM3SparseMSAImpl
return MiniMaxM3SparseBackend, MiniMaxM3SparseTritonImpl
def select_main_impl_cls(
*,
topk_blocks: int,
kv_cache_dtype: str,
num_kv_heads: int,
) -> type[MiniMaxM3SparseImpl]:
"""Backward-compatible implementation-only selector."""
return select_main_backend_and_impl_cls(
topk_blocks=topk_blocks,
kv_cache_dtype=kv_cache_dtype,
num_kv_heads=num_kv_heads,
)[1]
+28 -6
View File
@@ -79,7 +79,7 @@ from vllm.models.minimax_m3.common.mm_preprocess import (
from vllm.models.minimax_m3.common.sparse_attention import (
MiniMaxM3SparseBackend,
MiniMaxM3SparseImpl,
select_main_impl_cls,
select_main_backend_and_impl_cls,
)
from vllm.models.minimax_m3.common.vision_tower import MiniMaxVLVisionModel
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -505,15 +505,15 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
# the attend impl reads them back (so nothing crosses the eager break as a
# Python value, which would freeze at capture).
self.topk_indices_buffer = topk_indices_buffer
self.attn_backend = MiniMaxM3SparseBackend
# Indexer (top-k selection) and main attention are separate impls, each
# picking Triton vs MSA off its cache dtype. impl is AttentionImplBase
# (broader than the AttentionImpl that AttentionLayerBase annotates).
self.impl: MiniMaxM3SparseImpl = select_main_impl_cls( # type: ignore[assignment]
self.attn_backend, impl_cls = select_main_backend_and_impl_cls(
topk_blocks=sparse_cfg["sparse_topk_blocks"],
kv_cache_dtype=self.kv_cache_dtype,
num_kv_heads=self.num_kv_heads,
)(
)
self.impl: MiniMaxM3SparseImpl = impl_cls( # type: ignore[assignment]
self.num_heads,
self.head_dim,
self.scaling,
@@ -521,6 +521,9 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
kv_cache_dtype=self.kv_cache_dtype,
topk_blocks=sparse_cfg["sparse_topk_blocks"],
sparse_block_size=sparse_cfg["sparse_block_size"],
msa_decode_backend=(
vllm_config.attention_config.minimax_m3_msa_decode_backend
),
)
# Self-contained nn.Module: owns its side cache, selects its impl in init.
self.indexer = MiniMaxM3Indexer(
@@ -594,6 +597,16 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
main_slot_mapping = fwd_slot_mapping[self.layer_name]
index_slot_mapping = fwd_slot_mapping[self.indexer.index_cache.prefix]
q = qkv.new_empty((num_tokens, self.q_size))
use_msa_decode = self.impl.should_use_msa_decode(self.layer_name)
query_fp8 = (
torch.empty(
(num_tokens, self.q_size),
dtype=torch.float8_e4m3fn,
device=qkv.device,
)
if use_msa_decode
else None
)
# index_q matches the index-K cache dtype (e4m3 for the fp8 score path);
# the fused kernel emits fp8 directly when this buffer is e4m3.
index_q = qkv.new_empty(
@@ -621,10 +634,12 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
q,
index_q,
self.kv_cache_dtype,
q_fp8_out=query_fp8,
q_fp8_scale=self._q_scale_float,
)
output = torch.empty_like(q)
attn_output = self._run_attention(q, index_q, output)
attn_output = self._run_attention(q, query_fp8, index_q, output)
output, _ = self.o_proj(attn_output)
return output
@@ -632,6 +647,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
def _run_attention(
self,
query: torch.Tensor,
query_fp8: torch.Tensor | None,
index_query: torch.Tensor,
output: torch.Tensor,
) -> torch.Tensor:
@@ -639,7 +655,13 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
# metadata and can't be captured into a cudagraph. The indexer writes its
# top-k into the shared ``topk_indices_buffer``; the attend reads it back.
self.indexer(index_query)
return self.impl.forward(self, query, self.kv_cache, output)
return self.impl.forward(
self,
query,
self.kv_cache,
output,
query_fp8=query_fp8,
)
class MiniMaxM3DecoderLayer(nn.Module):
@@ -0,0 +1,316 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MiniMax CUTLASS sparse decode using per-query-token page indices."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
import torch
from vllm.config.attention import MiniMaxM3MSADecodeBackend
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
_MAX_NUM_Q_HEADS = 64
_MAX_NUM_KV_HEADS = 4
_HEAD_DIM = 128
_PAGE_SIZE = 128
_TOPK = 16
# fmha_sm100 plans one row per query head. Keep every cached plan within the
# fixed planner allocation used by the MSA decode kernel.
_MAX_QUERY_HEAD_ROWS = 65536
_MAX_DECODE_QUERY_LEN = 32
# Kernel benchmarks put the CUTLASS crossover at 16 requests for TP1 and TP4.
_MIN_CUTLASS_BATCH_SIZE = 16
@dataclass
class MSACutlassDecodeMetadata:
plan: Any
page_table: torch.Tensor
@triton.jit
def _update_runtime_metadata_kernel(
seq_lens_ptr,
kv_segment_lens_ptr,
qo_offset_ptr,
num_rows: tl.constexpr,
decode_query_len: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < num_rows
request = offsets // decode_query_len
local_query = offsets % decode_query_len
seq_len = tl.load(seq_lens_ptr + request, mask=mask)
tl.store(kv_segment_lens_ptr + offsets, seq_len, mask=mask)
tl.store(
qo_offset_ptr + offsets,
seq_len - decode_query_len + local_query,
mask=mask,
)
@dataclass
class MSACutlassDecodePlanCache:
"""Reusable plans whose mutable tensors retain cudagraph-stable addresses."""
plans: dict[tuple[int, ...], Any] = field(init=False, default_factory=dict)
def _build_plan(
self,
*,
batch: int,
decode_query_len: int,
page_table_stride: int,
initial_seq_lens_cpu: torch.Tensor,
device: torch.device,
num_q_heads: int,
num_kv_heads: int,
page_size: int,
topk_blocks: int,
) -> Any:
from vllm.third_party.fmha_sm100.api import fmha_sm100_plan
qo_lens_cpu = torch.full((batch,), decode_query_len, dtype=torch.int32)
kv_lens_cpu = initial_seq_lens_cpu
plan = fmha_sm100_plan(
qo_lens_cpu,
kv_lens_cpu,
num_q_heads,
num_kv_heads=num_kv_heads,
qo_offset=kv_lens_cpu - qo_lens_cpu,
page_size=page_size,
output_maxscore=False,
kv_block_num=topk_blocks,
causal=True,
sparse_kernel_mode="decode",
use_fp8_kvcache=True,
split_prefill_decode=False,
device=device,
)
plan_info = plan[3]
row_starts = (
torch.arange(batch, dtype=torch.int32, device=device)
.mul_(page_table_stride)
.repeat_interleave(decode_query_len)
)
page_indptr = torch.cat(
(
row_starts,
torch.tensor(
[batch * page_table_stride],
dtype=torch.int32,
device=device,
),
)
)
plan_info["kv_page_indptr"].copy_(page_indptr)
return plan
def prepare(
self,
block_table: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_cpu: torch.Tensor,
decode_query_len: int,
*,
num_q_heads: int,
num_kv_heads: int,
page_size: int,
topk_blocks: int,
) -> MSACutlassDecodeMetadata:
batch = int(seq_lens.shape[0])
if (
block_table.device.type != "cuda"
or block_table.dtype != torch.int32
or not block_table.is_contiguous()
or block_table.shape[0] != batch
):
raise ValueError(
"MSA sparse decode requires a contiguous CUDA int32 block "
"table with one row per request"
)
if (
seq_lens.dtype != torch.int32
or not seq_lens.is_contiguous()
or seq_lens.device != block_table.device
):
raise ValueError(
"MSA sparse decode requires contiguous CUDA int32 sequence "
"lengths on the block table device"
)
if (
seq_lens_cpu.device.type != "cpu"
or seq_lens_cpu.dtype != torch.int32
or not seq_lens_cpu.is_contiguous()
or seq_lens_cpu.shape != seq_lens.shape
):
raise ValueError(
"MSA sparse decode requires contiguous CPU int32 sequence "
"lengths matching the device sequence lengths"
)
page_table_stride = int(block_table.stride(0))
key = (
batch,
decode_query_len,
page_table_stride,
num_q_heads,
num_kv_heads,
page_size,
topk_blocks,
)
plan = self.plans.get(key)
if plan is None:
plan = self._build_plan(
batch=batch,
decode_query_len=decode_query_len,
page_table_stride=page_table_stride,
initial_seq_lens_cpu=seq_lens_cpu,
device=seq_lens.device,
num_q_heads=num_q_heads,
num_kv_heads=num_kv_heads,
page_size=page_size,
topk_blocks=topk_blocks,
)
self.plans[key] = plan
plan_info = plan[3]
num_rows = batch * decode_query_len
_update_runtime_metadata_kernel[(triton.cdiv(num_rows, 128),)](
seq_lens,
plan_info["kv_segment_lens"],
plan_info["qo_offset"],
num_rows=num_rows,
decode_query_len=decode_query_len,
BLOCK_SIZE=128,
)
return MSACutlassDecodeMetadata(
plan=plan,
page_table=block_table.view(-1),
)
def _supported_head_geometry(num_q_heads: int, num_kv_heads: int) -> bool:
return (
0 < num_q_heads <= _MAX_NUM_Q_HEADS
and 0 < num_kv_heads <= _MAX_NUM_KV_HEADS
and num_q_heads % num_kv_heads == 0
)
def supports_cutlass_sparse_decode(
*,
decode_backend: MiniMaxM3MSADecodeBackend,
num_q_heads: int,
num_kv_heads: int,
kv_cache_dtype: str,
page_size: int,
topk_blocks: int,
) -> bool:
"""Return whether static model geometry supports CUTLASS sparse decode."""
return (
decode_backend == "cutlass"
and current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
and kv_cache_dtype in ("fp8", "fp8_e4m3")
and _supported_head_geometry(num_q_heads, num_kv_heads)
and page_size == _PAGE_SIZE
and topk_blocks == _TOPK
)
def should_prepare_decode_metadata(
batch_size: int,
decode_query_len: int,
*,
decode_backend: MiniMaxM3MSADecodeBackend,
num_q_heads: int,
num_kv_heads: int,
kv_cache_dtype: str,
page_size: int,
topk_blocks: int,
) -> bool:
"""Return whether a graph shape can use the CUTLASS decode path."""
total_q = batch_size * decode_query_len
return (
supports_cutlass_sparse_decode(
decode_backend=decode_backend,
num_q_heads=num_q_heads,
num_kv_heads=num_kv_heads,
kv_cache_dtype=kv_cache_dtype,
page_size=page_size,
topk_blocks=topk_blocks,
)
and 1 <= decode_query_len <= _MAX_DECODE_QUERY_LEN
and batch_size >= _MIN_CUTLASS_BATCH_SIZE
and total_q * num_q_heads <= _MAX_QUERY_HEAD_ROWS
)
@torch.no_grad()
def prepare_decode_metadata(
block_table: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_cpu: torch.Tensor,
decode_query_len: int,
*,
num_q_heads: int,
num_kv_heads: int,
page_size: int,
topk_blocks: int,
plan_cache: MSACutlassDecodePlanCache | None = None,
) -> MSACutlassDecodeMetadata:
"""Prepare graph-stable runtime metadata for one sparse decode step."""
cache = plan_cache or MSACutlassDecodePlanCache()
return cache.prepare(
block_table,
seq_lens,
seq_lens_cpu,
decode_query_len,
num_q_heads=num_q_heads,
num_kv_heads=num_kv_heads,
page_size=page_size,
topk_blocks=topk_blocks,
)
@torch.no_grad()
def msa_cutlass_sparse_decode(
query_fp8: torch.Tensor,
kv_cache: torch.Tensor,
topk: torch.Tensor,
output: torch.Tensor,
metadata: MSACutlassDecodeMetadata,
*,
scale: float,
q_scale_float: float,
k_scale_float: float,
v_scale_float: float,
) -> None:
"""Run CUTLASS sparse decode with metadata prepared by the MSA builder."""
key, value = kv_cache.split(_HEAD_DIM, dim=-1)
from vllm.third_party.fmha_sm100.api import fmha_sm100
fmha_sm100(
query_fp8,
key,
value,
metadata.plan,
kv_indices=metadata.page_table,
kv_block_indexes=topk,
out=output,
output_maxscore=False,
output_o=True,
sm_scale=scale,
q_scale=q_scale_float,
k_scale=k_scale_float,
v_scale=v_scale_float,
o_scale=1.0,
)
@@ -1,28 +1,198 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MSA (SM100/Blackwell) block-sparse attend for MiniMax M3.
"""MSA (SM100/Blackwell) block-sparse attention for MiniMax M3.
Prefill attends with ``fmha_sm100`` (``build_k2q_csr`` + ``sparse_atten_func``);
decode falls back to the Triton split-K kernel (no MSA decode yet). ``fmha_sm100``
imports are function-local, so this module is import-safe on AMD/non-SM100.
Prefill attends with ``fmha_sm100`` (``build_k2q_csr`` + ``sparse_atten_func``).
Decode uses Triton split-K by default, with an opt-in CUTLASS ``fmha_sm100``
path for regular decode and speculative verification.
"""
from dataclasses import dataclass
import torch
from vllm.config import VllmConfig
from vllm.config.attention import MiniMaxM3MSADecodeBackend
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.models.minimax_m3.common.ops.sparse_attn import (
SPARSE_BLOCK_SIZE,
minimax_m3_sparse_attn_decode,
)
from vllm.models.minimax_m3.common.sparse_attention import (
MiniMaxM3SparseBackend,
MiniMaxM3SparseDecodeMetadata,
MiniMaxM3SparseImpl,
MiniMaxM3SparseMetadata,
MiniMaxM3SparseMetadataBuilder,
)
from vllm.v1.attention.backend import AttentionLayer
from vllm.models.minimax_m3.nvidia.msa_cutlass_sparse_decode import (
MSACutlassDecodeMetadata,
MSACutlassDecodePlanCache,
msa_cutlass_sparse_decode,
prepare_decode_metadata,
should_prepare_decode_metadata,
supports_cutlass_sparse_decode,
)
from vllm.v1.attention.backend import (
AttentionLayer,
CommonAttentionMetadata,
)
from vllm.v1.kv_cache_interface import AttentionSpec
logger = init_logger(__name__)
class MiniMaxM3SparseMSABackend(MiniMaxM3SparseBackend):
"""MiniMax M3 backend with NVIDIA MSA-specific decode metadata."""
@staticmethod
def get_builder_cls() -> type["MiniMaxM3SparseMSAMetadataBuilder"]:
return MiniMaxM3SparseMSAMetadataBuilder
class MiniMaxM3SparseCutlassBackend(MiniMaxM3SparseMSABackend):
"""Attention-backend alias selecting CUTLASS MSA sparse decode."""
@staticmethod
def get_name() -> str:
return "CUTLASS_MSA"
class MiniMaxM3SparseTritonBackend(MiniMaxM3SparseMSABackend):
"""Attention-backend alias selecting Triton MSA sparse decode."""
@staticmethod
def get_name() -> str:
return "TRITON_MSA"
@dataclass
class MiniMaxM3SparseMSADecodeMetadata(MiniMaxM3SparseDecodeMetadata):
msa_cutlass: MSACutlassDecodeMetadata | None = None
class MiniMaxM3SparseMSAMetadataBuilder(MiniMaxM3SparseMetadataBuilder):
"""Prepare MSA plans only for decode shapes supported by ``fmha_sm100``."""
def __init__(
self,
kv_cache_spec: AttentionSpec,
layer_names: list[str],
vllm_config: VllmConfig,
device: torch.device,
) -> None:
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
config = vllm_config.model_config.hf_text_config
tp_size = vllm_config.parallel_config.tensor_parallel_size
self.num_q_heads = config.num_attention_heads // tp_size
self.num_kv_heads = kv_cache_spec.num_kv_heads
self.topk_blocks = config.sparse_attention_config["sparse_topk_blocks"]
# AttentionSpec stores every FP8 mode as uint8, so retain the configured
# format to distinguish E4M3 (supported) from E5M2 before planning.
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
self.decode_backend = vllm_config.attention_config.minimax_m3_msa_decode_backend
self.msa_cutlass_plan_cache = MSACutlassDecodePlanCache()
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> MiniMaxM3SparseMetadata:
metadata = super().build(
common_prefix_len,
common_attn_metadata,
fast_build,
)
decode = metadata.decode
if decode is None:
return metadata
msa_cutlass = None
if should_prepare_decode_metadata(
metadata.num_decodes,
decode.decode_query_len,
decode_backend=self.decode_backend,
num_q_heads=self.num_q_heads,
num_kv_heads=self.num_kv_heads,
kv_cache_dtype=self.kv_cache_dtype,
page_size=SPARSE_BLOCK_SIZE,
topk_blocks=self.topk_blocks,
):
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
assert seq_lens_cpu is not None
msa_cutlass = prepare_decode_metadata(
decode.block_table,
decode.seq_lens,
seq_lens_cpu[: metadata.num_decodes],
decode.decode_query_len,
num_q_heads=self.num_q_heads,
num_kv_heads=self.num_kv_heads,
page_size=SPARSE_BLOCK_SIZE,
topk_blocks=self.topk_blocks,
plan_cache=self.msa_cutlass_plan_cache,
)
metadata.decode = MiniMaxM3SparseMSADecodeMetadata(
seq_lens=decode.seq_lens,
block_table=decode.block_table,
decode_query_len=decode.decode_query_len,
msa_cutlass=msa_cutlass,
)
return metadata
class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
"""MSA block-sparse attend (``fmha_sm100``); Triton split-K decode."""
"""MSA block-sparse attention with guarded CUTLASS sparse decode."""
def __init__(
self,
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: int | None = None,
kv_cache_dtype: str = "auto",
*,
topk_blocks: int,
sparse_block_size: int,
msa_decode_backend: MiniMaxM3MSADecodeBackend = "triton",
) -> None:
super().__init__(
num_heads,
head_size,
scale,
num_kv_heads,
kv_cache_dtype,
topk_blocks=topk_blocks,
sparse_block_size=sparse_block_size,
)
self.use_cutlass_decode = supports_cutlass_sparse_decode(
decode_backend=msa_decode_backend,
num_q_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
kv_cache_dtype=self.kv_cache_dtype,
page_size=self.block_size,
topk_blocks=self.topk_blocks,
)
logger.info_once(
"MiniMax M3 MSA sparse decode selected %s",
"CUTLASS" if self.use_cutlass_decode else "Triton",
)
def should_use_msa_decode(self, layer_name: str) -> bool:
if not self.use_cutlass_decode:
return False
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
return False
main_md = attn_metadata[layer_name]
if not isinstance(main_md, MiniMaxM3SparseMetadata):
return False
decode = main_md.decode
return (
isinstance(decode, MiniMaxM3SparseMSADecodeMetadata)
and decode.msa_cutlass is not None
)
def forward(
self,
@@ -30,6 +200,8 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
query: torch.Tensor,
kv_cache: torch.Tensor,
output: torch.Tensor,
*,
query_fp8: torch.Tensor | None = None,
) -> torch.Tensor:
attn_metadata = get_forward_context().attn_metadata
if not isinstance(attn_metadata, dict):
@@ -52,23 +224,42 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
k_scale = getattr(layer, "_k_scale", None) if self.use_fp8_kv else None
v_scale = getattr(layer, "_v_scale", None) if self.use_fp8_kv else None
# Decode [:nd]: Triton split-K placeholder (no MSA decode yet).
# Decode [:nd]: CUTLASS for planned shapes, otherwise Triton.
if main_md.num_decodes > 0:
d = main_md.decode
assert d is not None
minimax_m3_sparse_attn_decode(
q[:nd],
kv_cache,
topk[:nd].transpose(0, 1),
d.block_table,
d.seq_lens,
self.num_kv_heads,
self.scale,
out[:nd],
d.decode_query_len,
k_scale=k_scale,
v_scale=v_scale,
msa_metadata = (
d.msa_cutlass
if isinstance(d, MiniMaxM3SparseMSADecodeMetadata)
else None
)
if self.use_cutlass_decode and msa_metadata is not None:
assert query_fp8 is not None
msa_cutlass_sparse_decode(
query_fp8[:nd].view(-1, self.num_heads, hd),
kv_cache,
topk[:nd],
out[:nd],
msa_metadata,
scale=self.scale,
q_scale_float=getattr(layer, "_q_scale_float", 1.0),
k_scale_float=getattr(layer, "_k_scale_float", 1.0),
v_scale_float=getattr(layer, "_v_scale_float", 1.0),
)
else:
minimax_m3_sparse_attn_decode(
q[:nd],
kv_cache,
topk[:nd].transpose(0, 1),
d.block_table,
d.seq_lens,
self.num_kv_heads,
self.scale,
out[:nd],
d.decode_query_len,
k_scale=k_scale,
v_scale=v_scale,
)
# Prefill [nd:]: MSA sparse FMHA over the selected blocks.
if main_md.num_prefills > 0:
+8
View File
@@ -101,6 +101,14 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
MINIMAX_M3_SPARSE = (
"vllm.models.minimax_m3.common.sparse_attention.MiniMaxM3SparseBackend"
)
CUTLASS_MSA = (
"vllm.models.minimax_m3.nvidia.sparse_attention_msa."
"MiniMaxM3SparseCutlassBackend"
)
TRITON_MSA = (
"vllm.models.minimax_m3.nvidia.sparse_attention_msa."
"MiniMaxM3SparseTritonBackend"
)
NO_ATTENTION = "vllm.v1.attention.backends.no_attention.NoAttentionBackend"
FLEX_ATTENTION = "vllm.v1.attention.backends.flex_attention.FlexAttentionBackend"
# HPC Attention Backend: