[Attention][MiniMax-M3] NVFP4 KV cache on the MSA sparse attention path (#59300)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-10-01 14:39:55 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent ea84051819
commit 7566d83bd3
12 changed files with 1297 additions and 73 deletions
@@ -56,6 +56,7 @@ from vllm.v1.attention.backends.utils import (
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheLayout,
get_kv_quant_mode,
is_quantized_kv_cache,
)
@@ -531,10 +532,10 @@ def select_main_backend_and_impl_cls(
"""Pick the main attention backend and implementation.
Blackwell (SM100) uses the MSA attend for supported top-k block counts
when the KV cache is BF16 or FP8 E4M3; MI355 uses AITER sparse PA
when the KV cache is BF16, FP8 E4M3 or NVFP4; MI355 uses AITER sparse PA
with shuffle KV cache layout; Other platforms and FP8 E5M2 fall
back to Triton. The MSA modules are imported lazily to avoid import errors
on unsupported platforms.
back to Triton. NVFP4 is MSA-only. The MSA modules are imported lazily to
avoid import errors on unsupported platforms.
"""
use_aiter_sparse_pa = minimax_m3_use_aiter_sparse_pa(
num_kv_heads, emits_sparse_block_table=emits_sparse_block_table
@@ -545,6 +546,18 @@ def select_main_backend_and_impl_cls(
and topk_blocks in (4, 8, 16, 32)
and kv_cache_dtype != "fp8_e5m2"
)
use_nvfp4 = get_kv_quant_mode(kv_cache_dtype).is_nvfp4
if use_nvfp4 and kv_cache_dtype != "nvfp4":
raise ValueError(
"MiniMax M3 sparse attention supports only the 'nvfp4' NVFP4 KV "
f"cache dtype, got {kv_cache_dtype}"
)
if use_nvfp4 and (use_aiter_sparse_pa or not use_msa):
raise ValueError(
"MiniMax M3 sparse attention supports an NVFP4 KV cache only with "
f"the SM100 MSA backend (kv_cache_dtype={kv_cache_dtype}, "
f"topk_blocks={topk_blocks})"
)
selected = (
"AITER_SPARSE_PA" if use_aiter_sparse_pa else ("MSA" if use_msa else "Triton")
)
@@ -565,8 +578,11 @@ def select_main_backend_and_impl_cls(
from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
MiniMaxM3SparseMSABackend,
MiniMaxM3SparseMSAImpl,
MiniMaxM3SparseMSANvfp4Backend,
)
if use_nvfp4:
return MiniMaxM3SparseMSANvfp4Backend, MiniMaxM3SparseMSAImpl
return MiniMaxM3SparseMSABackend, MiniMaxM3SparseMSAImpl
return MiniMaxM3SparseBackend, MiniMaxM3SparseTritonImpl
+7 -1
View File
@@ -500,6 +500,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
# of wrapping the generic Attention module. Keep the same runtime scale
# attributes so FP8 KV reads can honor vLLM's per-layer descale contract.
set_default_quant_scales(self, register_buffer=True)
# The fused insert quantizes k/v into NVFP4 pages with these scales.
self.use_nvfp4_kv = get_kv_quant_mode(self.kv_cache_dtype).is_nvfp4
# Indexer side-cache dtype, mirroring --kv-cache-dtype for the main
# cache (--attention-config '{"indexer_kv_dtype": ...}').
self.indexer_kv_dtype = vllm_config.attention_config.resolve_indexer_kv_dtype(
@@ -611,6 +613,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
main_slot_mapping = fwd_slot_mapping[self.layer_name]
index_slot_mapping = fwd_slot_mapping[self.indexer.index_cache.prefix]
q = qkv.new_empty((num_tokens, self.q_size))
# With query_fp8, q is emitted only in fp8; the attend dequantizes it
# into ``q`` for the tokens that run on bf16-query kernels.
query_fp8 = self._allocate_query_fp8(qkv)
# index_q matches the index-K cache dtype (e4m3 for the fp8 score path);
# the fused kernel emits fp8 directly when this buffer is e4m3.
@@ -636,11 +640,13 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
self.kv_cache,
self.indexer.index_cache.kv_cache,
self.kv_cache.size(2), # paged-cache block size
q,
q if query_fp8 is None else None,
index_q,
self.kv_cache_dtype,
q_fp8_out=query_fp8,
q_fp8_scale=self._q_scale_float,
kv_k_scale=self._k_scale if self.use_nvfp4_kv else None,
kv_v_scale=self._v_scale if self.use_nvfp4_kv else None,
)
output = torch.empty_like(q)
@@ -12,6 +12,7 @@ import torch
from vllm.config.attention import MiniMaxM3MSADecodeBackend
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.kv_cache_interface import get_kv_quant_mode
_MAX_NUM_Q_HEADS = 64
_MAX_NUM_KV_HEADS = 4
@@ -23,9 +24,36 @@ _TOPK = 16
_MAX_QUERY_HEAD_ROWS = 65536
_MAX_DECODE_QUERY_LEN = 32
# Kernel benchmarks put the CUTLASS crossover at 16 requests for TP1 and TP4.
# NVFP4 has no Triton fallback, so it takes CUTLASS at every batch size.
_MIN_CUTLASS_BATCH_SIZE = 16
def is_nvfp4_kv_cache(kv_cache_dtype: str) -> bool:
return get_kv_quant_mode(kv_cache_dtype).is_nvfp4
def nvfp4_kv_cache_views(
kv_cache: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Return the K data, K scale, V data and V scale views of an NVFP4 cache.
``kv_cache`` is uint8 ``[pages, 2 * Hkv, page_size, 72]``; slot
``2 * head + side`` (K = 0, V = 1) is that head's E2M1 data
``[page_size, 64]`` followed by its E4M3 block scales ``[page_size, 8]``,
the one page layout ``fmha_sm100`` reads.
"""
from vllm.third_party.fmha_sm100.nvfp4_kv import nvfp4_head_slot_views
return nvfp4_head_slot_views(*nvfp4_kv_cache_slots(kv_cache))
def nvfp4_kv_cache_slots(
kv_cache: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Return the per-head K and V slots of an NVFP4 cache."""
return kv_cache[:, 0::2], kv_cache[:, 1::2]
@dataclass
class MSACutlassDecodeMetadata:
plan: Any
@@ -91,6 +119,9 @@ class MSACutlassDecodePlanCache:
use_fp8_kvcache=True,
split_prefill_decode=False,
device=device,
# The Q8KV4 route keeps plan-time lengths and its own page offsets,
# which this cached plan does not refresh.
decode_backend="kv_mode3",
)
plan_info = plan[3]
@@ -214,11 +245,12 @@ def supports_cutlass_sparse_decode(
topk_blocks: int,
) -> bool:
"""Return whether static model geometry supports CUTLASS sparse decode."""
nvfp4 = is_nvfp4_kv_cache(kv_cache_dtype)
return (
decode_backend == "cutlass"
(decode_backend == "cutlass" or nvfp4)
and current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
and kv_cache_dtype in ("fp8", "fp8_e4m3")
and (kv_cache_dtype in ("fp8", "fp8_e4m3") or nvfp4)
and _supported_head_geometry(num_q_heads, num_kv_heads)
and page_size == _PAGE_SIZE
and topk_blocks == _TOPK
@@ -248,7 +280,7 @@ def should_prepare_decode_metadata(
topk_blocks=topk_blocks,
)
and 1 <= decode_query_len <= _MAX_DECODE_QUERY_LEN
and batch_size >= _MIN_CUTLASS_BATCH_SIZE
and (batch_size >= _MIN_CUTLASS_BATCH_SIZE or is_nvfp4_kv_cache(kv_cache_dtype))
and total_q * num_q_heads <= _MAX_QUERY_HEAD_ROWS
)
@@ -292,9 +324,22 @@ def msa_cutlass_sparse_decode(
q_scale_float: float,
k_scale_float: float,
v_scale_float: float,
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
) -> None:
"""Run CUTLASS sparse decode with metadata prepared by the MSA builder."""
key, value = kv_cache.split(_HEAD_DIM, dim=-1)
"""Run CUTLASS sparse decode with metadata prepared by the MSA builder.
NVFP4 caches are uint8 head-slot pages (see ``nvfp4_kv_cache_views``) and
take the device global scales ``k_scale``/``v_scale``.
"""
if kv_cache.dtype == torch.uint8:
assert k_scale is not None and v_scale is not None
key, value = nvfp4_kv_cache_slots(kv_cache)
k_scale_arg: float | torch.Tensor = k_scale
v_scale_arg: float | torch.Tensor = v_scale
else:
key, value = kv_cache.split(_HEAD_DIM, dim=-1)
k_scale_arg, v_scale_arg = k_scale_float, v_scale_float
from vllm.third_party.fmha_sm100.api import fmha_sm100
@@ -310,7 +355,7 @@ def msa_cutlass_sparse_decode(
output_o=True,
sm_scale=scale,
q_scale=q_scale_float,
k_scale=k_scale_float,
v_scale=v_scale_float,
k_scale=k_scale_arg,
v_scale=v_scale_arg,
o_scale=1.0,
)
@@ -4,15 +4,18 @@
Prefill attends with ``fmha_sm100`` (``build_k2q_csr`` + ``sparse_atten_func``).
Decode uses Triton split-K by default, with an opt-in CUTLASS ``fmha_sm100``
path for regular decode and speculative verification.
path for regular decode and speculative verification. NVFP4 KV caches read the
vLLM packed pages directly in both prefill and CUTLASS decode.
"""
from dataclasses import dataclass
from dataclasses import dataclass, replace
from typing import ClassVar
import torch
from vllm.config import VllmConfig
from vllm.config.attention import MiniMaxM3MSADecodeBackend
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.models.minimax_m3.common.ops.sparse_attn import (
@@ -29,20 +32,33 @@ from vllm.models.minimax_m3.common.sparse_attention import (
from vllm.models.minimax_m3.nvidia.msa_cutlass_sparse_decode import (
MSACutlassDecodeMetadata,
MSACutlassDecodePlanCache,
is_nvfp4_kv_cache,
msa_cutlass_sparse_decode,
nvfp4_kv_cache_views,
prepare_decode_metadata,
should_prepare_decode_metadata,
supports_cutlass_sparse_decode,
)
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
from vllm.v1.attention.backend import (
AttentionLayer,
CommonAttentionMetadata,
)
from vllm.v1.kv_cache_interface import AttentionSpec
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
logger = init_logger(__name__)
def _dequantize_query(
query_fp8: torch.Tensor, q_scale: float, out: torch.Tensor
) -> torch.Tensor:
"""Write ``query_fp8 * q_scale`` into ``out`` (E4M3 is exact in BF16)."""
out.copy_(query_fp8)
if q_scale != 1.0:
out.mul_(q_scale)
return out
class MiniMaxM3SparseMSABackend(MiniMaxM3SparseBackend):
"""MiniMax M3 backend with NVIDIA MSA-specific decode metadata."""
@@ -51,6 +67,30 @@ class MiniMaxM3SparseMSABackend(MiniMaxM3SparseBackend):
return MiniMaxM3SparseMSAMetadataBuilder
class MiniMaxM3SparseMSANvfp4Backend(MiniMaxM3SparseMSABackend):
"""MSA backend over HND NVFP4 KV cache pages in per-head K/V slots."""
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["nvfp4"]
@classmethod
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
"""Store each head's K and V as two slots of packed data + block scales
(see ``nvfp4_kv_cache_views``)."""
if spec.state_content_bytes is not None or not spec.kv_quant_mode.is_nvfp4:
return spec
return replace(
spec,
num_head_slots=2 * spec.num_kv_heads,
state_content_bytes=nvfp4_kv_cache_full_dim(spec.head_size)
* get_dtype_size(spec.dtype),
)
@classmethod
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...] | None:
# Every head's K/V slots are one contiguous run of the page.
return (KVCacheLayout.LBHNC,)
class MiniMaxM3SparseCutlassBackend(MiniMaxM3SparseMSABackend):
"""Attention-backend alias selecting CUTLASS MSA sparse decode."""
@@ -166,6 +206,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
topk_blocks=topk_blocks,
sparse_block_size=sparse_block_size,
)
self.use_nvfp4_kv = is_nvfp4_kv_cache(kv_cache_dtype)
self.use_cutlass_decode = supports_cutlass_sparse_decode(
decode_backend=msa_decode_backend,
num_q_heads=self.num_heads,
@@ -174,6 +215,12 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
page_size=self.block_size,
topk_blocks=self.topk_blocks,
)
if self.use_nvfp4_kv and not self.use_cutlass_decode:
raise ValueError(
"MiniMax M3 NVFP4 KV cache requires CUTLASS MSA sparse decode "
f"(num_heads={self.num_heads}, num_kv_heads={self.num_kv_heads}, "
f"block_size={self.block_size}, topk_blocks={self.topk_blocks})"
)
logger.info_once(
"MiniMax M3 MSA sparse decode selected %s",
"CUTLASS" if self.use_cutlass_decode else "Triton",
@@ -218,9 +265,17 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
hd = self.head_size
q = query[:num_tokens].view(-1, self.num_heads, hd)
out = output[:num_tokens].view(-1, self.num_heads, hd)
kv_cache = (
kv_cache.view(self.kv_cache_fp8_dtype) if self.use_fp8_kv else kv_cache
# Given query_fp8, it is the only q the fused insert wrote: CUTLASS
# decode reads it directly, and the bf16-query kernels read it
# dequantized into ``query``.
q_fp8 = (
None
if query_fp8 is None
else query_fp8[:num_tokens].view(-1, self.num_heads, hd)
)
q_scale_float = getattr(layer, "_q_scale_float", 1.0)
if self.use_fp8_kv and not self.use_nvfp4_kv:
kv_cache = kv_cache.view(self.kv_cache_fp8_dtype)
k_scale = getattr(layer, "_k_scale", None) if self.use_fp8_kv else None
v_scale = getattr(layer, "_v_scale", None) if self.use_fp8_kv else None
@@ -234,21 +289,31 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
else None
)
if self.use_cutlass_decode and msa_metadata is not None:
assert query_fp8 is not None
assert q_fp8 is not None
msa_cutlass_sparse_decode(
query_fp8[:nd].view(-1, self.num_heads, hd),
q_fp8[:nd],
kv_cache,
topk[:nd],
out[:nd],
msa_metadata,
scale=self.scale,
q_scale_float=getattr(layer, "_q_scale_float", 1.0),
q_scale_float=q_scale_float,
k_scale_float=getattr(layer, "_k_scale_float", 1.0),
v_scale_float=getattr(layer, "_v_scale_float", 1.0),
k_scale=k_scale if self.use_nvfp4_kv else None,
v_scale=v_scale if self.use_nvfp4_kv else None,
)
elif self.use_nvfp4_kv:
raise RuntimeError(
"MiniMax M3 NVFP4 KV cache has no Triton decode fallback; "
f"CUTLASS decode was not planned for {main_md.num_decodes} "
f"requests with query length {d.decode_query_len}"
)
else:
minimax_m3_sparse_attn_decode(
q[:nd],
q[:nd]
if q_fp8 is None
else _dequantize_query(q_fp8[:nd], q_scale_float, q[:nd]),
kv_cache,
topk[:nd].transpose(0, 1),
d.block_table,
@@ -266,6 +331,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
from vllm.third_party.fmha_sm100.sparse import (
build_k2q_csr,
sparse_atten_func,
sparse_atten_nvfp4_kv_func,
)
p = main_md.prefill
@@ -273,8 +339,11 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
# [H, prefill, MK] transposed view; build_k2q_csr consumes the
# strided view directly (topK stays innermost-contiguous).
prefill_topk = topk[nd:num_tokens].transpose(0, 1)
qp = q[nd:]
k_cache, v_cache = kv_cache.split(self.head_size, dim=-1)
qp = (
q[nd:]
if q_fp8 is None
else _dequantize_query(q_fp8[nd:], q_scale_float, q[nd:])
)
k2q_row_ptr, k2q_q_indices, schedule = build_k2q_csr(
prefill_topk,
p.cu_seqlens_q,
@@ -287,6 +356,34 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
qhead_per_kv=qp.shape[1] // self.num_kv_heads,
return_schedule=True,
)
if self.use_nvfp4_kv:
k_data, k_sf, v_data, v_sf = nvfp4_kv_cache_views(kv_cache)
sparse_atten_nvfp4_kv_func(
qp,
k_data,
v_data,
k_sf,
v_sf,
k_scale,
v_scale,
k2q_row_ptr,
k2q_q_indices,
self.topk_blocks,
cu_seqlens_q=p.cu_seqlens_q,
cu_seqlens_k=p.cu_seqlens_k,
max_seqlen_q=p.max_query_len,
max_seqlen_k=p.max_seq_len,
blk_kv=SPARSE_BLOCK_SIZE,
causal=True,
softmax_scale=self.scale,
page_table=p.block_table,
seqused_k=p.seq_lens,
schedule=schedule,
out=out[nd:],
kv_layout="vllm",
)
return output
k_cache, v_cache = kv_cache.split(self.head_size, dim=-1)
sparse_atten_func(
qp,
k_cache,