[Attention][MiniMax-M3] Add MSA speculative decode verification (#50032)
This commit is contained in:
@@ -2655,6 +2655,8 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
index_q_out: torch.Tensor | None = None,
|
||||
kv_cache_dtype: str = "auto",
|
||||
skip_index_branch: bool = False,
|
||||
q_fp8_out: torch.Tensor | None = None,
|
||||
q_fp8_scale: float = 1.0,
|
||||
) -> None:
|
||||
"""Fused MiniMax-M3 attention pre-processing (in-place).
|
||||
|
||||
@@ -2677,6 +2679,9 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
callers skip a separate ``.contiguous()`` copy before the SM100 sparse
|
||||
attention's flat TMA descriptor.
|
||||
|
||||
If ``q_fp8_out`` is given, the same normalized q is also written in FP8
|
||||
E4M3 using ``q_fp8_scale`` as its dequantization scale.
|
||||
|
||||
When ``skip_index_branch`` is true, sparse rows still keep their packed
|
||||
``[index_q | index_k]`` tail, but the kernel only processes the main q/k/v
|
||||
branches and main KV cache. This is used by MiniMax-M3 index-topk reuse
|
||||
@@ -2704,6 +2709,8 @@ def fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
index_q_out,
|
||||
kv_cache_dtype,
|
||||
skip_index_branch,
|
||||
q_fp8_out,
|
||||
q_fp8_scale,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnu
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
IndexerKVDType = Literal["bf16", "fp8", "mxfp4", "nvfp4"]
|
||||
MiniMaxM3MSADecodeBackend = Literal["triton", "cutlass"]
|
||||
|
||||
|
||||
@config
|
||||
@@ -20,6 +21,9 @@ class AttentionConfig:
|
||||
backend: AttentionBackendEnum | None = None
|
||||
"""Attention backend to use. Use "auto" or None for automatic selection."""
|
||||
|
||||
minimax_m3_msa_decode_backend: MiniMaxM3MSADecodeBackend = "triton"
|
||||
"""Sparse decode kernel used by the MiniMax M3 MSA backend."""
|
||||
|
||||
backend_per_kind: dict[str, AttentionBackendEnum] = field(default_factory=dict)
|
||||
"""Per-KV-cache-group attention backend overrides, keyed by
|
||||
`KVCacheSpecKind` (e.g. `{"mla_attention": "FLASHINFER_MLA",
|
||||
@@ -99,6 +103,17 @@ class AttentionConfig:
|
||||
PyTorch >= 2.9, and 128 for encoder-only attention or older PyTorch
|
||||
versions."""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
msa_aliases: dict[AttentionBackendEnum, MiniMaxM3MSADecodeBackend] = {
|
||||
AttentionBackendEnum.CUTLASS_MSA: "cutlass",
|
||||
AttentionBackendEnum.TRITON_MSA: "triton",
|
||||
}
|
||||
if self.backend in msa_aliases:
|
||||
self.minimax_m3_msa_decode_backend = msa_aliases[self.backend]
|
||||
# The alias selects only MiniMax's sparse decode kernel. Dense
|
||||
# layers still use the platform's normal automatic backend.
|
||||
self.backend = None
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Provide a hash that uniquely identifies all the configs
|
||||
|
||||
@@ -23,6 +23,8 @@ class MiniMaxM3SparseAiterPAImpl(MiniMaxM3SparseImpl):
|
||||
query: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
*,
|
||||
query_fp8: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.models.minimax_m3.amd.ops.sparse_pa import (
|
||||
minimax_m3_sparse_attn_decode_aiter,
|
||||
|
||||
@@ -330,6 +330,7 @@ class MiniMaxM3SparseImpl(AttentionImplBase[MiniMaxM3SparseMetadata]):
|
||||
*,
|
||||
topk_blocks: int,
|
||||
sparse_block_size: int,
|
||||
msa_decode_backend: str = "triton",
|
||||
) -> None:
|
||||
self.num_heads = num_heads
|
||||
self.head_size = head_size
|
||||
@@ -355,6 +356,8 @@ class MiniMaxM3SparseImpl(AttentionImplBase[MiniMaxM3SparseMetadata]):
|
||||
query: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
*,
|
||||
query_fp8: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Attend the queries to the indexer-selected blocks. Per kernel.
|
||||
|
||||
@@ -364,6 +367,9 @@ class MiniMaxM3SparseImpl(AttentionImplBase[MiniMaxM3SparseMetadata]):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def should_use_msa_decode(self, layer_name: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class MiniMaxM3SparseTritonImpl(MiniMaxM3SparseImpl):
|
||||
"""Triton block-sparse attend (``minimax_m3_sparse_attn``) + Triton decode."""
|
||||
@@ -374,6 +380,8 @@ class MiniMaxM3SparseTritonImpl(MiniMaxM3SparseImpl):
|
||||
query: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
*,
|
||||
query_fp8: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
@@ -442,18 +450,18 @@ class MiniMaxM3SparseTritonImpl(MiniMaxM3SparseImpl):
|
||||
return output
|
||||
|
||||
|
||||
def select_main_impl_cls(
|
||||
def select_main_backend_and_impl_cls(
|
||||
*,
|
||||
topk_blocks: int,
|
||||
kv_cache_dtype: str,
|
||||
num_kv_heads: int,
|
||||
) -> type[MiniMaxM3SparseImpl]:
|
||||
"""Pick the main attend impl off the main KV-cache dtype.
|
||||
) -> tuple[type[MiniMaxM3SparseBackend], type[MiniMaxM3SparseImpl]]:
|
||||
"""Pick the main attention backend and implementation.
|
||||
|
||||
Blackwell (SM100) uses the MSA attend for supported top-k block counts
|
||||
when the KV cache is BF16 or FP8 E4M3; MI355 uses AITER sparse PA
|
||||
with shuffle KV cache layout; Other platforms and FP8 E5M2 fall
|
||||
back to Triton. The MSA modules are imported lazily avoid import errors
|
||||
back to Triton. The MSA modules are imported lazily to avoid import errors
|
||||
on unsupported platforms.
|
||||
"""
|
||||
use_aiter_sparse_pa = minimax_m3_use_aiter_sparse_pa(num_kv_heads)
|
||||
@@ -477,11 +485,26 @@ def select_main_impl_cls(
|
||||
MiniMaxM3SparseAiterPAImpl,
|
||||
)
|
||||
|
||||
return MiniMaxM3SparseAiterPAImpl
|
||||
return MiniMaxM3SparseBackend, MiniMaxM3SparseAiterPAImpl
|
||||
if use_msa:
|
||||
from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
|
||||
MiniMaxM3SparseMSABackend,
|
||||
MiniMaxM3SparseMSAImpl,
|
||||
)
|
||||
|
||||
return MiniMaxM3SparseMSAImpl
|
||||
return MiniMaxM3SparseTritonImpl
|
||||
return MiniMaxM3SparseMSABackend, MiniMaxM3SparseMSAImpl
|
||||
return MiniMaxM3SparseBackend, MiniMaxM3SparseTritonImpl
|
||||
|
||||
|
||||
def select_main_impl_cls(
|
||||
*,
|
||||
topk_blocks: int,
|
||||
kv_cache_dtype: str,
|
||||
num_kv_heads: int,
|
||||
) -> type[MiniMaxM3SparseImpl]:
|
||||
"""Backward-compatible implementation-only selector."""
|
||||
return select_main_backend_and_impl_cls(
|
||||
topk_blocks=topk_blocks,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
num_kv_heads=num_kv_heads,
|
||||
)[1]
|
||||
|
||||
@@ -79,7 +79,7 @@ from vllm.models.minimax_m3.common.mm_preprocess import (
|
||||
from vllm.models.minimax_m3.common.sparse_attention import (
|
||||
MiniMaxM3SparseBackend,
|
||||
MiniMaxM3SparseImpl,
|
||||
select_main_impl_cls,
|
||||
select_main_backend_and_impl_cls,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.vision_tower import MiniMaxVLVisionModel
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
@@ -505,15 +505,15 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
# the attend impl reads them back (so nothing crosses the eager break as a
|
||||
# Python value, which would freeze at capture).
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
self.attn_backend = MiniMaxM3SparseBackend
|
||||
# Indexer (top-k selection) and main attention are separate impls, each
|
||||
# picking Triton vs MSA off its cache dtype. impl is AttentionImplBase
|
||||
# (broader than the AttentionImpl that AttentionLayerBase annotates).
|
||||
self.impl: MiniMaxM3SparseImpl = select_main_impl_cls( # type: ignore[assignment]
|
||||
self.attn_backend, impl_cls = select_main_backend_and_impl_cls(
|
||||
topk_blocks=sparse_cfg["sparse_topk_blocks"],
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
)(
|
||||
)
|
||||
self.impl: MiniMaxM3SparseImpl = impl_cls( # type: ignore[assignment]
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
@@ -521,6 +521,9 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
topk_blocks=sparse_cfg["sparse_topk_blocks"],
|
||||
sparse_block_size=sparse_cfg["sparse_block_size"],
|
||||
msa_decode_backend=(
|
||||
vllm_config.attention_config.minimax_m3_msa_decode_backend
|
||||
),
|
||||
)
|
||||
# Self-contained nn.Module: owns its side cache, selects its impl in init.
|
||||
self.indexer = MiniMaxM3Indexer(
|
||||
@@ -594,6 +597,16 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
main_slot_mapping = fwd_slot_mapping[self.layer_name]
|
||||
index_slot_mapping = fwd_slot_mapping[self.indexer.index_cache.prefix]
|
||||
q = qkv.new_empty((num_tokens, self.q_size))
|
||||
use_msa_decode = self.impl.should_use_msa_decode(self.layer_name)
|
||||
query_fp8 = (
|
||||
torch.empty(
|
||||
(num_tokens, self.q_size),
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=qkv.device,
|
||||
)
|
||||
if use_msa_decode
|
||||
else None
|
||||
)
|
||||
# index_q matches the index-K cache dtype (e4m3 for the fp8 score path);
|
||||
# the fused kernel emits fp8 directly when this buffer is e4m3.
|
||||
index_q = qkv.new_empty(
|
||||
@@ -621,10 +634,12 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
q,
|
||||
index_q,
|
||||
self.kv_cache_dtype,
|
||||
q_fp8_out=query_fp8,
|
||||
q_fp8_scale=self._q_scale_float,
|
||||
)
|
||||
|
||||
output = torch.empty_like(q)
|
||||
attn_output = self._run_attention(q, index_q, output)
|
||||
attn_output = self._run_attention(q, query_fp8, index_q, output)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
@@ -632,6 +647,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
def _run_attention(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
query_fp8: torch.Tensor | None,
|
||||
index_query: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
@@ -639,7 +655,13 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
# metadata and can't be captured into a cudagraph. The indexer writes its
|
||||
# top-k into the shared ``topk_indices_buffer``; the attend reads it back.
|
||||
self.indexer(index_query)
|
||||
return self.impl.forward(self, query, self.kv_cache, output)
|
||||
return self.impl.forward(
|
||||
self,
|
||||
query,
|
||||
self.kv_cache,
|
||||
output,
|
||||
query_fp8=query_fp8,
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3DecoderLayer(nn.Module):
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiniMax CUTLASS sparse decode using per-query-token page indices."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.attention import MiniMaxM3MSADecodeBackend
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_MAX_NUM_Q_HEADS = 64
|
||||
_MAX_NUM_KV_HEADS = 4
|
||||
_HEAD_DIM = 128
|
||||
_PAGE_SIZE = 128
|
||||
_TOPK = 16
|
||||
# fmha_sm100 plans one row per query head. Keep every cached plan within the
|
||||
# fixed planner allocation used by the MSA decode kernel.
|
||||
_MAX_QUERY_HEAD_ROWS = 65536
|
||||
_MAX_DECODE_QUERY_LEN = 32
|
||||
# Kernel benchmarks put the CUTLASS crossover at 16 requests for TP1 and TP4.
|
||||
_MIN_CUTLASS_BATCH_SIZE = 16
|
||||
|
||||
|
||||
@dataclass
|
||||
class MSACutlassDecodeMetadata:
|
||||
plan: Any
|
||||
page_table: torch.Tensor
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _update_runtime_metadata_kernel(
|
||||
seq_lens_ptr,
|
||||
kv_segment_lens_ptr,
|
||||
qo_offset_ptr,
|
||||
num_rows: tl.constexpr,
|
||||
decode_query_len: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < num_rows
|
||||
request = offsets // decode_query_len
|
||||
local_query = offsets % decode_query_len
|
||||
seq_len = tl.load(seq_lens_ptr + request, mask=mask)
|
||||
tl.store(kv_segment_lens_ptr + offsets, seq_len, mask=mask)
|
||||
tl.store(
|
||||
qo_offset_ptr + offsets,
|
||||
seq_len - decode_query_len + local_query,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MSACutlassDecodePlanCache:
|
||||
"""Reusable plans whose mutable tensors retain cudagraph-stable addresses."""
|
||||
|
||||
plans: dict[tuple[int, ...], Any] = field(init=False, default_factory=dict)
|
||||
|
||||
def _build_plan(
|
||||
self,
|
||||
*,
|
||||
batch: int,
|
||||
decode_query_len: int,
|
||||
page_table_stride: int,
|
||||
initial_seq_lens_cpu: torch.Tensor,
|
||||
device: torch.device,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
page_size: int,
|
||||
topk_blocks: int,
|
||||
) -> Any:
|
||||
from vllm.third_party.fmha_sm100.api import fmha_sm100_plan
|
||||
|
||||
qo_lens_cpu = torch.full((batch,), decode_query_len, dtype=torch.int32)
|
||||
kv_lens_cpu = initial_seq_lens_cpu
|
||||
plan = fmha_sm100_plan(
|
||||
qo_lens_cpu,
|
||||
kv_lens_cpu,
|
||||
num_q_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
qo_offset=kv_lens_cpu - qo_lens_cpu,
|
||||
page_size=page_size,
|
||||
output_maxscore=False,
|
||||
kv_block_num=topk_blocks,
|
||||
causal=True,
|
||||
sparse_kernel_mode="decode",
|
||||
use_fp8_kvcache=True,
|
||||
split_prefill_decode=False,
|
||||
device=device,
|
||||
)
|
||||
|
||||
plan_info = plan[3]
|
||||
row_starts = (
|
||||
torch.arange(batch, dtype=torch.int32, device=device)
|
||||
.mul_(page_table_stride)
|
||||
.repeat_interleave(decode_query_len)
|
||||
)
|
||||
page_indptr = torch.cat(
|
||||
(
|
||||
row_starts,
|
||||
torch.tensor(
|
||||
[batch * page_table_stride],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
),
|
||||
)
|
||||
)
|
||||
plan_info["kv_page_indptr"].copy_(page_indptr)
|
||||
return plan
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
block_table: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
decode_query_len: int,
|
||||
*,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
page_size: int,
|
||||
topk_blocks: int,
|
||||
) -> MSACutlassDecodeMetadata:
|
||||
batch = int(seq_lens.shape[0])
|
||||
if (
|
||||
block_table.device.type != "cuda"
|
||||
or block_table.dtype != torch.int32
|
||||
or not block_table.is_contiguous()
|
||||
or block_table.shape[0] != batch
|
||||
):
|
||||
raise ValueError(
|
||||
"MSA sparse decode requires a contiguous CUDA int32 block "
|
||||
"table with one row per request"
|
||||
)
|
||||
if (
|
||||
seq_lens.dtype != torch.int32
|
||||
or not seq_lens.is_contiguous()
|
||||
or seq_lens.device != block_table.device
|
||||
):
|
||||
raise ValueError(
|
||||
"MSA sparse decode requires contiguous CUDA int32 sequence "
|
||||
"lengths on the block table device"
|
||||
)
|
||||
if (
|
||||
seq_lens_cpu.device.type != "cpu"
|
||||
or seq_lens_cpu.dtype != torch.int32
|
||||
or not seq_lens_cpu.is_contiguous()
|
||||
or seq_lens_cpu.shape != seq_lens.shape
|
||||
):
|
||||
raise ValueError(
|
||||
"MSA sparse decode requires contiguous CPU int32 sequence "
|
||||
"lengths matching the device sequence lengths"
|
||||
)
|
||||
|
||||
page_table_stride = int(block_table.stride(0))
|
||||
key = (
|
||||
batch,
|
||||
decode_query_len,
|
||||
page_table_stride,
|
||||
num_q_heads,
|
||||
num_kv_heads,
|
||||
page_size,
|
||||
topk_blocks,
|
||||
)
|
||||
plan = self.plans.get(key)
|
||||
if plan is None:
|
||||
plan = self._build_plan(
|
||||
batch=batch,
|
||||
decode_query_len=decode_query_len,
|
||||
page_table_stride=page_table_stride,
|
||||
initial_seq_lens_cpu=seq_lens_cpu,
|
||||
device=seq_lens.device,
|
||||
num_q_heads=num_q_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
page_size=page_size,
|
||||
topk_blocks=topk_blocks,
|
||||
)
|
||||
self.plans[key] = plan
|
||||
|
||||
plan_info = plan[3]
|
||||
num_rows = batch * decode_query_len
|
||||
_update_runtime_metadata_kernel[(triton.cdiv(num_rows, 128),)](
|
||||
seq_lens,
|
||||
plan_info["kv_segment_lens"],
|
||||
plan_info["qo_offset"],
|
||||
num_rows=num_rows,
|
||||
decode_query_len=decode_query_len,
|
||||
BLOCK_SIZE=128,
|
||||
)
|
||||
return MSACutlassDecodeMetadata(
|
||||
plan=plan,
|
||||
page_table=block_table.view(-1),
|
||||
)
|
||||
|
||||
|
||||
def _supported_head_geometry(num_q_heads: int, num_kv_heads: int) -> bool:
|
||||
return (
|
||||
0 < num_q_heads <= _MAX_NUM_Q_HEADS
|
||||
and 0 < num_kv_heads <= _MAX_NUM_KV_HEADS
|
||||
and num_q_heads % num_kv_heads == 0
|
||||
)
|
||||
|
||||
|
||||
def supports_cutlass_sparse_decode(
|
||||
*,
|
||||
decode_backend: MiniMaxM3MSADecodeBackend,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
kv_cache_dtype: str,
|
||||
page_size: int,
|
||||
topk_blocks: int,
|
||||
) -> bool:
|
||||
"""Return whether static model geometry supports CUTLASS sparse decode."""
|
||||
return (
|
||||
decode_backend == "cutlass"
|
||||
and current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
and kv_cache_dtype in ("fp8", "fp8_e4m3")
|
||||
and _supported_head_geometry(num_q_heads, num_kv_heads)
|
||||
and page_size == _PAGE_SIZE
|
||||
and topk_blocks == _TOPK
|
||||
)
|
||||
|
||||
|
||||
def should_prepare_decode_metadata(
|
||||
batch_size: int,
|
||||
decode_query_len: int,
|
||||
*,
|
||||
decode_backend: MiniMaxM3MSADecodeBackend,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
kv_cache_dtype: str,
|
||||
page_size: int,
|
||||
topk_blocks: int,
|
||||
) -> bool:
|
||||
"""Return whether a graph shape can use the CUTLASS decode path."""
|
||||
total_q = batch_size * decode_query_len
|
||||
return (
|
||||
supports_cutlass_sparse_decode(
|
||||
decode_backend=decode_backend,
|
||||
num_q_heads=num_q_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
page_size=page_size,
|
||||
topk_blocks=topk_blocks,
|
||||
)
|
||||
and 1 <= decode_query_len <= _MAX_DECODE_QUERY_LEN
|
||||
and batch_size >= _MIN_CUTLASS_BATCH_SIZE
|
||||
and total_q * num_q_heads <= _MAX_QUERY_HEAD_ROWS
|
||||
)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def prepare_decode_metadata(
|
||||
block_table: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
decode_query_len: int,
|
||||
*,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
page_size: int,
|
||||
topk_blocks: int,
|
||||
plan_cache: MSACutlassDecodePlanCache | None = None,
|
||||
) -> MSACutlassDecodeMetadata:
|
||||
"""Prepare graph-stable runtime metadata for one sparse decode step."""
|
||||
cache = plan_cache or MSACutlassDecodePlanCache()
|
||||
return cache.prepare(
|
||||
block_table,
|
||||
seq_lens,
|
||||
seq_lens_cpu,
|
||||
decode_query_len,
|
||||
num_q_heads=num_q_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
page_size=page_size,
|
||||
topk_blocks=topk_blocks,
|
||||
)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def msa_cutlass_sparse_decode(
|
||||
query_fp8: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
topk: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
metadata: MSACutlassDecodeMetadata,
|
||||
*,
|
||||
scale: float,
|
||||
q_scale_float: float,
|
||||
k_scale_float: float,
|
||||
v_scale_float: float,
|
||||
) -> None:
|
||||
"""Run CUTLASS sparse decode with metadata prepared by the MSA builder."""
|
||||
key, value = kv_cache.split(_HEAD_DIM, dim=-1)
|
||||
|
||||
from vllm.third_party.fmha_sm100.api import fmha_sm100
|
||||
|
||||
fmha_sm100(
|
||||
query_fp8,
|
||||
key,
|
||||
value,
|
||||
metadata.plan,
|
||||
kv_indices=metadata.page_table,
|
||||
kv_block_indexes=topk,
|
||||
out=output,
|
||||
output_maxscore=False,
|
||||
output_o=True,
|
||||
sm_scale=scale,
|
||||
q_scale=q_scale_float,
|
||||
k_scale=k_scale_float,
|
||||
v_scale=v_scale_float,
|
||||
o_scale=1.0,
|
||||
)
|
||||
@@ -1,28 +1,198 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MSA (SM100/Blackwell) block-sparse attend for MiniMax M3.
|
||||
"""MSA (SM100/Blackwell) block-sparse attention for MiniMax M3.
|
||||
|
||||
Prefill attends with ``fmha_sm100`` (``build_k2q_csr`` + ``sparse_atten_func``);
|
||||
decode falls back to the Triton split-K kernel (no MSA decode yet). ``fmha_sm100``
|
||||
imports are function-local, so this module is import-safe on AMD/non-SM100.
|
||||
Prefill attends with ``fmha_sm100`` (``build_k2q_csr`` + ``sparse_atten_func``).
|
||||
Decode uses Triton split-K by default, with an opt-in CUTLASS ``fmha_sm100``
|
||||
path for regular decode and speculative verification.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.attention import MiniMaxM3MSADecodeBackend
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
||||
SPARSE_BLOCK_SIZE,
|
||||
minimax_m3_sparse_attn_decode,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.sparse_attention import (
|
||||
MiniMaxM3SparseBackend,
|
||||
MiniMaxM3SparseDecodeMetadata,
|
||||
MiniMaxM3SparseImpl,
|
||||
MiniMaxM3SparseMetadata,
|
||||
MiniMaxM3SparseMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionLayer
|
||||
from vllm.models.minimax_m3.nvidia.msa_cutlass_sparse_decode import (
|
||||
MSACutlassDecodeMetadata,
|
||||
MSACutlassDecodePlanCache,
|
||||
msa_cutlass_sparse_decode,
|
||||
prepare_decode_metadata,
|
||||
should_prepare_decode_metadata,
|
||||
supports_cutlass_sparse_decode,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionLayer,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MiniMaxM3SparseMSABackend(MiniMaxM3SparseBackend):
|
||||
"""MiniMax M3 backend with NVIDIA MSA-specific decode metadata."""
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["MiniMaxM3SparseMSAMetadataBuilder"]:
|
||||
return MiniMaxM3SparseMSAMetadataBuilder
|
||||
|
||||
|
||||
class MiniMaxM3SparseCutlassBackend(MiniMaxM3SparseMSABackend):
|
||||
"""Attention-backend alias selecting CUTLASS MSA sparse decode."""
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "CUTLASS_MSA"
|
||||
|
||||
|
||||
class MiniMaxM3SparseTritonBackend(MiniMaxM3SparseMSABackend):
|
||||
"""Attention-backend alias selecting Triton MSA sparse decode."""
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "TRITON_MSA"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MiniMaxM3SparseMSADecodeMetadata(MiniMaxM3SparseDecodeMetadata):
|
||||
msa_cutlass: MSACutlassDecodeMetadata | None = None
|
||||
|
||||
|
||||
class MiniMaxM3SparseMSAMetadataBuilder(MiniMaxM3SparseMetadataBuilder):
|
||||
"""Prepare MSA plans only for decode shapes supported by ``fmha_sm100``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
layer_names: list[str],
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
self.num_q_heads = config.num_attention_heads // tp_size
|
||||
self.num_kv_heads = kv_cache_spec.num_kv_heads
|
||||
self.topk_blocks = config.sparse_attention_config["sparse_topk_blocks"]
|
||||
# AttentionSpec stores every FP8 mode as uint8, so retain the configured
|
||||
# format to distinguish E4M3 (supported) from E5M2 before planning.
|
||||
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
|
||||
self.decode_backend = vllm_config.attention_config.minimax_m3_msa_decode_backend
|
||||
self.msa_cutlass_plan_cache = MSACutlassDecodePlanCache()
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> MiniMaxM3SparseMetadata:
|
||||
metadata = super().build(
|
||||
common_prefix_len,
|
||||
common_attn_metadata,
|
||||
fast_build,
|
||||
)
|
||||
decode = metadata.decode
|
||||
if decode is None:
|
||||
return metadata
|
||||
|
||||
msa_cutlass = None
|
||||
if should_prepare_decode_metadata(
|
||||
metadata.num_decodes,
|
||||
decode.decode_query_len,
|
||||
decode_backend=self.decode_backend,
|
||||
num_q_heads=self.num_q_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
page_size=SPARSE_BLOCK_SIZE,
|
||||
topk_blocks=self.topk_blocks,
|
||||
):
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
msa_cutlass = prepare_decode_metadata(
|
||||
decode.block_table,
|
||||
decode.seq_lens,
|
||||
seq_lens_cpu[: metadata.num_decodes],
|
||||
decode.decode_query_len,
|
||||
num_q_heads=self.num_q_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
page_size=SPARSE_BLOCK_SIZE,
|
||||
topk_blocks=self.topk_blocks,
|
||||
plan_cache=self.msa_cutlass_plan_cache,
|
||||
)
|
||||
metadata.decode = MiniMaxM3SparseMSADecodeMetadata(
|
||||
seq_lens=decode.seq_lens,
|
||||
block_table=decode.block_table,
|
||||
decode_query_len=decode.decode_query_len,
|
||||
msa_cutlass=msa_cutlass,
|
||||
)
|
||||
return metadata
|
||||
|
||||
|
||||
class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
"""MSA block-sparse attend (``fmha_sm100``); Triton split-K decode."""
|
||||
"""MSA block-sparse attention with guarded CUTLASS sparse decode."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
scale: float,
|
||||
num_kv_heads: int | None = None,
|
||||
kv_cache_dtype: str = "auto",
|
||||
*,
|
||||
topk_blocks: int,
|
||||
sparse_block_size: int,
|
||||
msa_decode_backend: MiniMaxM3MSADecodeBackend = "triton",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_heads,
|
||||
head_size,
|
||||
scale,
|
||||
num_kv_heads,
|
||||
kv_cache_dtype,
|
||||
topk_blocks=topk_blocks,
|
||||
sparse_block_size=sparse_block_size,
|
||||
)
|
||||
self.use_cutlass_decode = supports_cutlass_sparse_decode(
|
||||
decode_backend=msa_decode_backend,
|
||||
num_q_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
page_size=self.block_size,
|
||||
topk_blocks=self.topk_blocks,
|
||||
)
|
||||
logger.info_once(
|
||||
"MiniMax M3 MSA sparse decode selected %s",
|
||||
"CUTLASS" if self.use_cutlass_decode else "Triton",
|
||||
)
|
||||
|
||||
def should_use_msa_decode(self, layer_name: str) -> bool:
|
||||
if not self.use_cutlass_decode:
|
||||
return False
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
return False
|
||||
main_md = attn_metadata[layer_name]
|
||||
if not isinstance(main_md, MiniMaxM3SparseMetadata):
|
||||
return False
|
||||
decode = main_md.decode
|
||||
return (
|
||||
isinstance(decode, MiniMaxM3SparseMSADecodeMetadata)
|
||||
and decode.msa_cutlass is not None
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -30,6 +200,8 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
query: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
*,
|
||||
query_fp8: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if not isinstance(attn_metadata, dict):
|
||||
@@ -52,23 +224,42 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
k_scale = getattr(layer, "_k_scale", None) if self.use_fp8_kv else None
|
||||
v_scale = getattr(layer, "_v_scale", None) if self.use_fp8_kv else None
|
||||
|
||||
# Decode [:nd]: Triton split-K placeholder (no MSA decode yet).
|
||||
# Decode [:nd]: CUTLASS for planned shapes, otherwise Triton.
|
||||
if main_md.num_decodes > 0:
|
||||
d = main_md.decode
|
||||
assert d is not None
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q[:nd],
|
||||
kv_cache,
|
||||
topk[:nd].transpose(0, 1),
|
||||
d.block_table,
|
||||
d.seq_lens,
|
||||
self.num_kv_heads,
|
||||
self.scale,
|
||||
out[:nd],
|
||||
d.decode_query_len,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
msa_metadata = (
|
||||
d.msa_cutlass
|
||||
if isinstance(d, MiniMaxM3SparseMSADecodeMetadata)
|
||||
else None
|
||||
)
|
||||
if self.use_cutlass_decode and msa_metadata is not None:
|
||||
assert query_fp8 is not None
|
||||
msa_cutlass_sparse_decode(
|
||||
query_fp8[:nd].view(-1, self.num_heads, hd),
|
||||
kv_cache,
|
||||
topk[:nd],
|
||||
out[:nd],
|
||||
msa_metadata,
|
||||
scale=self.scale,
|
||||
q_scale_float=getattr(layer, "_q_scale_float", 1.0),
|
||||
k_scale_float=getattr(layer, "_k_scale_float", 1.0),
|
||||
v_scale_float=getattr(layer, "_v_scale_float", 1.0),
|
||||
)
|
||||
else:
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q[:nd],
|
||||
kv_cache,
|
||||
topk[:nd].transpose(0, 1),
|
||||
d.block_table,
|
||||
d.seq_lens,
|
||||
self.num_kv_heads,
|
||||
self.scale,
|
||||
out[:nd],
|
||||
d.decode_query_len,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
|
||||
# Prefill [nd:]: MSA sparse FMHA over the selected blocks.
|
||||
if main_md.num_prefills > 0:
|
||||
|
||||
@@ -101,6 +101,14 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
|
||||
MINIMAX_M3_SPARSE = (
|
||||
"vllm.models.minimax_m3.common.sparse_attention.MiniMaxM3SparseBackend"
|
||||
)
|
||||
CUTLASS_MSA = (
|
||||
"vllm.models.minimax_m3.nvidia.sparse_attention_msa."
|
||||
"MiniMaxM3SparseCutlassBackend"
|
||||
)
|
||||
TRITON_MSA = (
|
||||
"vllm.models.minimax_m3.nvidia.sparse_attention_msa."
|
||||
"MiniMaxM3SparseTritonBackend"
|
||||
)
|
||||
NO_ATTENTION = "vllm.v1.attention.backends.no_attention.NoAttentionBackend"
|
||||
FLEX_ATTENTION = "vllm.v1.attention.backends.flex_attention.FlexAttentionBackend"
|
||||
# HPC Attention Backend:
|
||||
|
||||
Reference in New Issue
Block a user