Signed-off-by: Anish Shanbhag <ashanbhag@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com>
415 lines
15 KiB
Python
415 lines
15 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""MSA (SM100/Blackwell) block-sparse attention for MiniMax M3.
|
|
|
|
Prefill attends with ``fmha_sm100`` (``build_k2q_csr`` + ``sparse_atten_func``).
|
|
Decode uses Triton split-K by default, with an opt-in CUTLASS ``fmha_sm100``
|
|
path for regular decode and speculative verification. NVFP4 KV caches read the
|
|
vLLM packed pages directly in both prefill and CUTLASS decode.
|
|
"""
|
|
|
|
import os
|
|
from dataclasses import dataclass, replace
|
|
from typing import ClassVar
|
|
|
|
import torch
|
|
|
|
from vllm.config import VllmConfig
|
|
from vllm.config.attention import MiniMaxM3MSADecodeBackend
|
|
from vllm.config.cache import CacheDType
|
|
from vllm.forward_context import get_forward_context
|
|
from vllm.logger import init_logger
|
|
from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
|
SPARSE_BLOCK_SIZE,
|
|
minimax_m3_sparse_attn_decode,
|
|
)
|
|
from vllm.models.minimax_m3.common.sparse_attention import (
|
|
MiniMaxM3SparseBackend,
|
|
MiniMaxM3SparseDecodeMetadata,
|
|
MiniMaxM3SparseImpl,
|
|
MiniMaxM3SparseMetadata,
|
|
MiniMaxM3SparseMetadataBuilder,
|
|
)
|
|
from vllm.models.minimax_m3.nvidia.msa_cutlass_sparse_decode import (
|
|
MSACutlassDecodeMetadata,
|
|
MSACutlassDecodePlanCache,
|
|
is_nvfp4_kv_cache,
|
|
msa_cutlass_sparse_decode,
|
|
nvfp4_kv_cache_views,
|
|
prepare_decode_metadata,
|
|
should_prepare_decode_metadata,
|
|
supports_cutlass_sparse_decode,
|
|
)
|
|
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
|
|
from vllm.v1.attention.backend import (
|
|
AttentionLayer,
|
|
CommonAttentionMetadata,
|
|
)
|
|
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
def _get_msa_cutlass_min_batch_size() -> int | None:
|
|
value = os.getenv("VLLM_MINIMAX_M3_MSA_CUTLASS_MIN_BATCH")
|
|
return int(value) if value is not None else None
|
|
|
|
|
|
def _dequantize_query(
|
|
query_fp8: torch.Tensor, q_scale: float, out: torch.Tensor
|
|
) -> torch.Tensor:
|
|
"""Write ``query_fp8 * q_scale`` into ``out`` (E4M3 is exact in BF16)."""
|
|
out.copy_(query_fp8)
|
|
if q_scale != 1.0:
|
|
out.mul_(q_scale)
|
|
return out
|
|
|
|
|
|
class MiniMaxM3SparseMSABackend(MiniMaxM3SparseBackend):
|
|
"""MiniMax M3 backend with NVIDIA MSA-specific decode metadata."""
|
|
|
|
@staticmethod
|
|
def get_builder_cls() -> type["MiniMaxM3SparseMSAMetadataBuilder"]:
|
|
return MiniMaxM3SparseMSAMetadataBuilder
|
|
|
|
|
|
class MiniMaxM3SparseMSANvfp4Backend(MiniMaxM3SparseMSABackend):
|
|
"""MSA backend over HND NVFP4 KV cache pages in per-head K/V slots."""
|
|
|
|
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["nvfp4"]
|
|
|
|
@classmethod
|
|
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
|
|
"""Store each head's K and V as two slots of packed data + block scales
|
|
(see ``nvfp4_kv_cache_views``)."""
|
|
if spec.state_content_bytes is not None or not spec.kv_quant_mode.is_nvfp4:
|
|
return spec
|
|
return replace(
|
|
spec,
|
|
num_head_slots=2 * spec.num_kv_heads,
|
|
state_content_bytes=nvfp4_kv_cache_full_dim(spec.head_size)
|
|
* get_dtype_size(spec.dtype),
|
|
)
|
|
|
|
@classmethod
|
|
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...] | None:
|
|
# Every head's K/V slots are one contiguous run of the page.
|
|
return (KVCacheLayout.LBHNC,)
|
|
|
|
|
|
class MiniMaxM3SparseCutlassBackend(MiniMaxM3SparseMSABackend):
|
|
"""Attention-backend alias selecting CUTLASS MSA sparse decode."""
|
|
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "CUTLASS_MSA"
|
|
|
|
|
|
class MiniMaxM3SparseTritonBackend(MiniMaxM3SparseMSABackend):
|
|
"""Attention-backend alias selecting Triton MSA sparse decode."""
|
|
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "TRITON_MSA"
|
|
|
|
|
|
@dataclass
|
|
class MiniMaxM3SparseMSADecodeMetadata(MiniMaxM3SparseDecodeMetadata):
|
|
msa_cutlass: MSACutlassDecodeMetadata | None = None
|
|
|
|
|
|
class MiniMaxM3SparseMSAMetadataBuilder(MiniMaxM3SparseMetadataBuilder):
|
|
"""Prepare MSA plans only for decode shapes supported by ``fmha_sm100``."""
|
|
|
|
def __init__(
|
|
self,
|
|
kv_cache_spec: AttentionSpec,
|
|
layer_names: list[str],
|
|
vllm_config: VllmConfig,
|
|
device: torch.device,
|
|
) -> None:
|
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
|
config = vllm_config.model_config.hf_text_config
|
|
tp_size = vllm_config.parallel_config.tensor_parallel_size
|
|
self.num_q_heads = config.num_attention_heads // tp_size
|
|
self.num_kv_heads = kv_cache_spec.num_kv_heads
|
|
self.topk_blocks = config.sparse_attention_config["sparse_topk_blocks"]
|
|
# AttentionSpec stores every FP8 mode as uint8, so retain the configured
|
|
# format to distinguish E4M3 (supported) from E5M2 before planning.
|
|
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
|
|
self.decode_backend = vllm_config.attention_config.minimax_m3_msa_decode_backend
|
|
self.msa_cutlass_plan_cache = MSACutlassDecodePlanCache()
|
|
self.msa_cutlass_min_batch_size = _get_msa_cutlass_min_batch_size()
|
|
|
|
def build(
|
|
self,
|
|
common_prefix_len: int,
|
|
common_attn_metadata: CommonAttentionMetadata,
|
|
fast_build: bool = False,
|
|
) -> MiniMaxM3SparseMetadata:
|
|
metadata = super().build(
|
|
common_prefix_len,
|
|
common_attn_metadata,
|
|
fast_build,
|
|
)
|
|
decode = metadata.decode
|
|
if decode is None:
|
|
return metadata
|
|
|
|
msa_cutlass = None
|
|
if should_prepare_decode_metadata(
|
|
metadata.num_decodes,
|
|
decode.decode_query_len,
|
|
decode_backend=self.decode_backend,
|
|
num_q_heads=self.num_q_heads,
|
|
num_kv_heads=self.num_kv_heads,
|
|
kv_cache_dtype=self.kv_cache_dtype,
|
|
page_size=SPARSE_BLOCK_SIZE,
|
|
topk_blocks=self.topk_blocks,
|
|
min_batch_size=self.msa_cutlass_min_batch_size,
|
|
):
|
|
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
|
assert seq_lens_cpu is not None
|
|
msa_cutlass = prepare_decode_metadata(
|
|
decode.block_table,
|
|
decode.seq_lens,
|
|
seq_lens_cpu[: metadata.num_decodes],
|
|
decode.decode_query_len,
|
|
num_q_heads=self.num_q_heads,
|
|
num_kv_heads=self.num_kv_heads,
|
|
page_size=SPARSE_BLOCK_SIZE,
|
|
topk_blocks=self.topk_blocks,
|
|
plan_cache=self.msa_cutlass_plan_cache,
|
|
)
|
|
metadata.decode = MiniMaxM3SparseMSADecodeMetadata(
|
|
seq_lens=decode.seq_lens,
|
|
block_table=decode.block_table,
|
|
decode_query_len=decode.decode_query_len,
|
|
msa_cutlass=msa_cutlass,
|
|
)
|
|
return metadata
|
|
|
|
|
|
class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
|
"""MSA block-sparse attention with guarded CUTLASS sparse decode."""
|
|
|
|
def __init__(
|
|
self,
|
|
num_heads: int,
|
|
head_size: int,
|
|
scale: float,
|
|
num_kv_heads: int | None = None,
|
|
kv_cache_dtype: str = "auto",
|
|
*,
|
|
topk_blocks: int,
|
|
sparse_block_size: int,
|
|
msa_decode_backend: MiniMaxM3MSADecodeBackend = "triton",
|
|
) -> None:
|
|
super().__init__(
|
|
num_heads,
|
|
head_size,
|
|
scale,
|
|
num_kv_heads,
|
|
kv_cache_dtype,
|
|
topk_blocks=topk_blocks,
|
|
sparse_block_size=sparse_block_size,
|
|
)
|
|
self.use_nvfp4_kv = is_nvfp4_kv_cache(kv_cache_dtype)
|
|
self.use_cutlass_decode = supports_cutlass_sparse_decode(
|
|
decode_backend=msa_decode_backend,
|
|
num_q_heads=self.num_heads,
|
|
num_kv_heads=self.num_kv_heads,
|
|
kv_cache_dtype=self.kv_cache_dtype,
|
|
page_size=self.block_size,
|
|
topk_blocks=self.topk_blocks,
|
|
)
|
|
if self.use_nvfp4_kv and not self.use_cutlass_decode:
|
|
raise ValueError(
|
|
"MiniMax M3 NVFP4 KV cache requires CUTLASS MSA sparse decode "
|
|
f"(num_heads={self.num_heads}, num_kv_heads={self.num_kv_heads}, "
|
|
f"block_size={self.block_size}, topk_blocks={self.topk_blocks})"
|
|
)
|
|
logger.info_once(
|
|
"MiniMax M3 MSA sparse decode selected %s",
|
|
"CUTLASS" if self.use_cutlass_decode else "Triton",
|
|
)
|
|
|
|
def should_use_msa_decode(self, layer_name: str) -> bool:
|
|
if not self.use_cutlass_decode:
|
|
return False
|
|
attn_metadata = get_forward_context().attn_metadata
|
|
if not isinstance(attn_metadata, dict):
|
|
return False
|
|
main_md = attn_metadata[layer_name]
|
|
if not isinstance(main_md, MiniMaxM3SparseMetadata):
|
|
return False
|
|
decode = main_md.decode
|
|
return (
|
|
isinstance(decode, MiniMaxM3SparseMSADecodeMetadata)
|
|
and decode.msa_cutlass is not None
|
|
)
|
|
|
|
def forward(
|
|
self,
|
|
layer: AttentionLayer,
|
|
query: torch.Tensor,
|
|
kv_cache: torch.Tensor,
|
|
output: torch.Tensor,
|
|
*,
|
|
query_fp8: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
attn_metadata = get_forward_context().attn_metadata
|
|
if not isinstance(attn_metadata, dict):
|
|
return output # profiling run; caches unbound
|
|
main_md = attn_metadata[layer.layer_name] # type: ignore[attr-defined]
|
|
assert isinstance(main_md, MiniMaxM3SparseMetadata)
|
|
|
|
nd = main_md.num_decode_tokens
|
|
num_tokens = main_md.num_actual_tokens
|
|
# Indexer top-k from the shared token-major buffer [total_q, H, MK]; the
|
|
# kernels want [H, tokens, MK], so slice tokens on dim 0 then transpose.
|
|
topk = layer.topk_indices_buffer # type: ignore[attr-defined]
|
|
assert topk is not None
|
|
hd = self.head_size
|
|
q = query[:num_tokens].view(-1, self.num_heads, hd)
|
|
out = output[:num_tokens].view(-1, self.num_heads, hd)
|
|
# Given query_fp8, it is the only q the fused insert wrote: CUTLASS
|
|
# decode and prefill read it directly, and Triton decode reads it
|
|
# dequantized into ``query``.
|
|
q_fp8 = (
|
|
None
|
|
if query_fp8 is None
|
|
else query_fp8[:num_tokens].view(-1, self.num_heads, hd)
|
|
)
|
|
q_scale_float = getattr(layer, "_q_scale_float", 1.0)
|
|
if self.use_fp8_kv and not self.use_nvfp4_kv:
|
|
kv_cache = kv_cache.view(self.kv_cache_fp8_dtype)
|
|
k_scale = getattr(layer, "_k_scale", None) if self.use_fp8_kv else None
|
|
v_scale = getattr(layer, "_v_scale", None) if self.use_fp8_kv else None
|
|
|
|
# Decode [:nd]: CUTLASS for planned shapes, otherwise Triton.
|
|
if main_md.num_decodes > 0:
|
|
d = main_md.decode
|
|
assert d is not None
|
|
msa_metadata = (
|
|
d.msa_cutlass
|
|
if isinstance(d, MiniMaxM3SparseMSADecodeMetadata)
|
|
else None
|
|
)
|
|
if self.use_cutlass_decode and msa_metadata is not None:
|
|
assert q_fp8 is not None
|
|
msa_cutlass_sparse_decode(
|
|
q_fp8[:nd],
|
|
kv_cache,
|
|
topk[:nd],
|
|
out[:nd],
|
|
msa_metadata,
|
|
scale=self.scale,
|
|
q_scale_float=q_scale_float,
|
|
k_scale_float=getattr(layer, "_k_scale_float", 1.0),
|
|
v_scale_float=getattr(layer, "_v_scale_float", 1.0),
|
|
k_scale=k_scale if self.use_nvfp4_kv else None,
|
|
v_scale=v_scale if self.use_nvfp4_kv else None,
|
|
)
|
|
elif self.use_nvfp4_kv:
|
|
raise RuntimeError(
|
|
"MiniMax M3 NVFP4 KV cache has no Triton decode fallback; "
|
|
f"CUTLASS decode was not planned for {main_md.num_decodes} "
|
|
f"requests with query length {d.decode_query_len}"
|
|
)
|
|
else:
|
|
minimax_m3_sparse_attn_decode(
|
|
q[:nd]
|
|
if q_fp8 is None
|
|
else _dequantize_query(q_fp8[:nd], q_scale_float, q[:nd]),
|
|
kv_cache,
|
|
topk[:nd].transpose(0, 1),
|
|
d.block_table,
|
|
d.seq_lens,
|
|
self.num_kv_heads,
|
|
self.scale,
|
|
out[:nd],
|
|
d.decode_query_len,
|
|
k_scale=k_scale,
|
|
v_scale=v_scale,
|
|
)
|
|
|
|
# Prefill [nd:]: MSA sparse FMHA over the selected blocks.
|
|
if main_md.num_prefills > 0:
|
|
from vllm.third_party.fmha_sm100.sparse import (
|
|
build_k2q_csr,
|
|
sparse_atten_func,
|
|
sparse_atten_nvfp4_kv_func,
|
|
)
|
|
|
|
p = main_md.prefill
|
|
assert p is not None
|
|
# [H, prefill, MK] transposed view; build_k2q_csr consumes the
|
|
# strided view directly (topK stays innermost-contiguous).
|
|
prefill_topk = topk[nd:num_tokens].transpose(0, 1)
|
|
# An E4M3 query attends as is, its scale folded into softmax_scale.
|
|
if q_fp8 is None:
|
|
qp, softmax_scale = q[nd:], self.scale
|
|
else:
|
|
qp, softmax_scale = q_fp8[nd:], self.scale * q_scale_float
|
|
k2q_row_ptr, k2q_q_indices, schedule = build_k2q_csr(
|
|
prefill_topk,
|
|
p.cu_seqlens_q,
|
|
p.cu_seqlens_k,
|
|
SPARSE_BLOCK_SIZE,
|
|
total_k=0,
|
|
max_seqlen_k=p.max_seq_len,
|
|
max_seqlen_q=p.max_query_len,
|
|
total_rows=p.total_kv_blocks,
|
|
qhead_per_kv=qp.shape[1] // self.num_kv_heads,
|
|
return_schedule=True,
|
|
)
|
|
if self.use_nvfp4_kv:
|
|
k_data, k_sf, v_data, v_sf = nvfp4_kv_cache_views(kv_cache)
|
|
sparse_atten_nvfp4_kv_func(
|
|
qp,
|
|
k_data,
|
|
v_data,
|
|
k_sf,
|
|
v_sf,
|
|
k_scale,
|
|
v_scale,
|
|
k2q_row_ptr,
|
|
k2q_q_indices,
|
|
self.topk_blocks,
|
|
cu_seqlens_q=p.cu_seqlens_q,
|
|
cu_seqlens_k=p.cu_seqlens_k,
|
|
max_seqlen_q=p.max_query_len,
|
|
max_seqlen_k=p.max_seq_len,
|
|
blk_kv=SPARSE_BLOCK_SIZE,
|
|
causal=True,
|
|
softmax_scale=softmax_scale,
|
|
page_table=p.block_table,
|
|
seqused_k=p.seq_lens,
|
|
schedule=schedule,
|
|
out=out[nd:],
|
|
kv_layout="vllm",
|
|
)
|
|
return output
|
|
k_cache, v_cache = kv_cache.split(self.head_size, dim=-1)
|
|
sparse_atten_func(
|
|
qp,
|
|
k_cache,
|
|
v_cache,
|
|
k2q_row_ptr,
|
|
k2q_q_indices,
|
|
topK=self.topk_blocks,
|
|
blk_kv=SPARSE_BLOCK_SIZE,
|
|
causal=True,
|
|
softmax_scale=softmax_scale,
|
|
cu_seqlens_q=p.cu_seqlens_q,
|
|
cu_seqlens_k=p.cu_seqlens_k,
|
|
max_seqlen_q=p.max_query_len,
|
|
max_seqlen_k=p.max_seq_len,
|
|
page_table=p.block_table,
|
|
seqused_k=p.seq_lens,
|
|
schedule=schedule,
|
|
out=out[nd:],
|
|
)
|
|
return output
|