[Attention][MiniMax-M3] NVFP4 KV cache on the MSA sparse attention path (#59300)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ea84051819
commit
7566d83bd3
@@ -56,6 +56,7 @@ from vllm.v1.attention.backends.utils import (
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheLayout,
|
||||
get_kv_quant_mode,
|
||||
is_quantized_kv_cache,
|
||||
)
|
||||
|
||||
@@ -531,10 +532,10 @@ def select_main_backend_and_impl_cls(
|
||||
"""Pick the main attention backend and implementation.
|
||||
|
||||
Blackwell (SM100) uses the MSA attend for supported top-k block counts
|
||||
when the KV cache is BF16 or FP8 E4M3; MI355 uses AITER sparse PA
|
||||
when the KV cache is BF16, FP8 E4M3 or NVFP4; MI355 uses AITER sparse PA
|
||||
with shuffle KV cache layout; Other platforms and FP8 E5M2 fall
|
||||
back to Triton. The MSA modules are imported lazily to avoid import errors
|
||||
on unsupported platforms.
|
||||
back to Triton. NVFP4 is MSA-only. The MSA modules are imported lazily to
|
||||
avoid import errors on unsupported platforms.
|
||||
"""
|
||||
use_aiter_sparse_pa = minimax_m3_use_aiter_sparse_pa(
|
||||
num_kv_heads, emits_sparse_block_table=emits_sparse_block_table
|
||||
@@ -545,6 +546,18 @@ def select_main_backend_and_impl_cls(
|
||||
and topk_blocks in (4, 8, 16, 32)
|
||||
and kv_cache_dtype != "fp8_e5m2"
|
||||
)
|
||||
use_nvfp4 = get_kv_quant_mode(kv_cache_dtype).is_nvfp4
|
||||
if use_nvfp4 and kv_cache_dtype != "nvfp4":
|
||||
raise ValueError(
|
||||
"MiniMax M3 sparse attention supports only the 'nvfp4' NVFP4 KV "
|
||||
f"cache dtype, got {kv_cache_dtype}"
|
||||
)
|
||||
if use_nvfp4 and (use_aiter_sparse_pa or not use_msa):
|
||||
raise ValueError(
|
||||
"MiniMax M3 sparse attention supports an NVFP4 KV cache only with "
|
||||
f"the SM100 MSA backend (kv_cache_dtype={kv_cache_dtype}, "
|
||||
f"topk_blocks={topk_blocks})"
|
||||
)
|
||||
selected = (
|
||||
"AITER_SPARSE_PA" if use_aiter_sparse_pa else ("MSA" if use_msa else "Triton")
|
||||
)
|
||||
@@ -565,8 +578,11 @@ def select_main_backend_and_impl_cls(
|
||||
from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
|
||||
MiniMaxM3SparseMSABackend,
|
||||
MiniMaxM3SparseMSAImpl,
|
||||
MiniMaxM3SparseMSANvfp4Backend,
|
||||
)
|
||||
|
||||
if use_nvfp4:
|
||||
return MiniMaxM3SparseMSANvfp4Backend, MiniMaxM3SparseMSAImpl
|
||||
return MiniMaxM3SparseMSABackend, MiniMaxM3SparseMSAImpl
|
||||
return MiniMaxM3SparseBackend, MiniMaxM3SparseTritonImpl
|
||||
|
||||
|
||||
@@ -500,6 +500,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
# of wrapping the generic Attention module. Keep the same runtime scale
|
||||
# attributes so FP8 KV reads can honor vLLM's per-layer descale contract.
|
||||
set_default_quant_scales(self, register_buffer=True)
|
||||
# The fused insert quantizes k/v into NVFP4 pages with these scales.
|
||||
self.use_nvfp4_kv = get_kv_quant_mode(self.kv_cache_dtype).is_nvfp4
|
||||
# Indexer side-cache dtype, mirroring --kv-cache-dtype for the main
|
||||
# cache (--attention-config '{"indexer_kv_dtype": ...}').
|
||||
self.indexer_kv_dtype = vllm_config.attention_config.resolve_indexer_kv_dtype(
|
||||
@@ -611,6 +613,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
main_slot_mapping = fwd_slot_mapping[self.layer_name]
|
||||
index_slot_mapping = fwd_slot_mapping[self.indexer.index_cache.prefix]
|
||||
q = qkv.new_empty((num_tokens, self.q_size))
|
||||
# With query_fp8, q is emitted only in fp8; the attend dequantizes it
|
||||
# into ``q`` for the tokens that run on bf16-query kernels.
|
||||
query_fp8 = self._allocate_query_fp8(qkv)
|
||||
# index_q matches the index-K cache dtype (e4m3 for the fp8 score path);
|
||||
# the fused kernel emits fp8 directly when this buffer is e4m3.
|
||||
@@ -636,11 +640,13 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
self.kv_cache,
|
||||
self.indexer.index_cache.kv_cache,
|
||||
self.kv_cache.size(2), # paged-cache block size
|
||||
q,
|
||||
q if query_fp8 is None else None,
|
||||
index_q,
|
||||
self.kv_cache_dtype,
|
||||
q_fp8_out=query_fp8,
|
||||
q_fp8_scale=self._q_scale_float,
|
||||
kv_k_scale=self._k_scale if self.use_nvfp4_kv else None,
|
||||
kv_v_scale=self._v_scale if self.use_nvfp4_kv else None,
|
||||
)
|
||||
|
||||
output = torch.empty_like(q)
|
||||
|
||||
@@ -12,6 +12,7 @@ import torch
|
||||
from vllm.config.attention import MiniMaxM3MSADecodeBackend
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.kv_cache_interface import get_kv_quant_mode
|
||||
|
||||
_MAX_NUM_Q_HEADS = 64
|
||||
_MAX_NUM_KV_HEADS = 4
|
||||
@@ -23,9 +24,36 @@ _TOPK = 16
|
||||
_MAX_QUERY_HEAD_ROWS = 65536
|
||||
_MAX_DECODE_QUERY_LEN = 32
|
||||
# Kernel benchmarks put the CUTLASS crossover at 16 requests for TP1 and TP4.
|
||||
# NVFP4 has no Triton fallback, so it takes CUTLASS at every batch size.
|
||||
_MIN_CUTLASS_BATCH_SIZE = 16
|
||||
|
||||
|
||||
def is_nvfp4_kv_cache(kv_cache_dtype: str) -> bool:
|
||||
return get_kv_quant_mode(kv_cache_dtype).is_nvfp4
|
||||
|
||||
|
||||
def nvfp4_kv_cache_views(
|
||||
kv_cache: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Return the K data, K scale, V data and V scale views of an NVFP4 cache.
|
||||
|
||||
``kv_cache`` is uint8 ``[pages, 2 * Hkv, page_size, 72]``; slot
|
||||
``2 * head + side`` (K = 0, V = 1) is that head's E2M1 data
|
||||
``[page_size, 64]`` followed by its E4M3 block scales ``[page_size, 8]``,
|
||||
the one page layout ``fmha_sm100`` reads.
|
||||
"""
|
||||
from vllm.third_party.fmha_sm100.nvfp4_kv import nvfp4_head_slot_views
|
||||
|
||||
return nvfp4_head_slot_views(*nvfp4_kv_cache_slots(kv_cache))
|
||||
|
||||
|
||||
def nvfp4_kv_cache_slots(
|
||||
kv_cache: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Return the per-head K and V slots of an NVFP4 cache."""
|
||||
return kv_cache[:, 0::2], kv_cache[:, 1::2]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MSACutlassDecodeMetadata:
|
||||
plan: Any
|
||||
@@ -91,6 +119,9 @@ class MSACutlassDecodePlanCache:
|
||||
use_fp8_kvcache=True,
|
||||
split_prefill_decode=False,
|
||||
device=device,
|
||||
# The Q8KV4 route keeps plan-time lengths and its own page offsets,
|
||||
# which this cached plan does not refresh.
|
||||
decode_backend="kv_mode3",
|
||||
)
|
||||
|
||||
plan_info = plan[3]
|
||||
@@ -214,11 +245,12 @@ def supports_cutlass_sparse_decode(
|
||||
topk_blocks: int,
|
||||
) -> bool:
|
||||
"""Return whether static model geometry supports CUTLASS sparse decode."""
|
||||
nvfp4 = is_nvfp4_kv_cache(kv_cache_dtype)
|
||||
return (
|
||||
decode_backend == "cutlass"
|
||||
(decode_backend == "cutlass" or nvfp4)
|
||||
and current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
and kv_cache_dtype in ("fp8", "fp8_e4m3")
|
||||
and (kv_cache_dtype in ("fp8", "fp8_e4m3") or nvfp4)
|
||||
and _supported_head_geometry(num_q_heads, num_kv_heads)
|
||||
and page_size == _PAGE_SIZE
|
||||
and topk_blocks == _TOPK
|
||||
@@ -248,7 +280,7 @@ def should_prepare_decode_metadata(
|
||||
topk_blocks=topk_blocks,
|
||||
)
|
||||
and 1 <= decode_query_len <= _MAX_DECODE_QUERY_LEN
|
||||
and batch_size >= _MIN_CUTLASS_BATCH_SIZE
|
||||
and (batch_size >= _MIN_CUTLASS_BATCH_SIZE or is_nvfp4_kv_cache(kv_cache_dtype))
|
||||
and total_q * num_q_heads <= _MAX_QUERY_HEAD_ROWS
|
||||
)
|
||||
|
||||
@@ -292,9 +324,22 @@ def msa_cutlass_sparse_decode(
|
||||
q_scale_float: float,
|
||||
k_scale_float: float,
|
||||
v_scale_float: float,
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Run CUTLASS sparse decode with metadata prepared by the MSA builder."""
|
||||
key, value = kv_cache.split(_HEAD_DIM, dim=-1)
|
||||
"""Run CUTLASS sparse decode with metadata prepared by the MSA builder.
|
||||
|
||||
NVFP4 caches are uint8 head-slot pages (see ``nvfp4_kv_cache_views``) and
|
||||
take the device global scales ``k_scale``/``v_scale``.
|
||||
"""
|
||||
if kv_cache.dtype == torch.uint8:
|
||||
assert k_scale is not None and v_scale is not None
|
||||
key, value = nvfp4_kv_cache_slots(kv_cache)
|
||||
k_scale_arg: float | torch.Tensor = k_scale
|
||||
v_scale_arg: float | torch.Tensor = v_scale
|
||||
else:
|
||||
key, value = kv_cache.split(_HEAD_DIM, dim=-1)
|
||||
k_scale_arg, v_scale_arg = k_scale_float, v_scale_float
|
||||
|
||||
from vllm.third_party.fmha_sm100.api import fmha_sm100
|
||||
|
||||
@@ -310,7 +355,7 @@ def msa_cutlass_sparse_decode(
|
||||
output_o=True,
|
||||
sm_scale=scale,
|
||||
q_scale=q_scale_float,
|
||||
k_scale=k_scale_float,
|
||||
v_scale=v_scale_float,
|
||||
k_scale=k_scale_arg,
|
||||
v_scale=v_scale_arg,
|
||||
o_scale=1.0,
|
||||
)
|
||||
|
||||
@@ -4,15 +4,18 @@
|
||||
|
||||
Prefill attends with ``fmha_sm100`` (``build_k2q_csr`` + ``sparse_atten_func``).
|
||||
Decode uses Triton split-K by default, with an opt-in CUTLASS ``fmha_sm100``
|
||||
path for regular decode and speculative verification.
|
||||
path for regular decode and speculative verification. NVFP4 KV caches read the
|
||||
vLLM packed pages directly in both prefill and CUTLASS decode.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.attention import MiniMaxM3MSADecodeBackend
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
||||
@@ -29,20 +32,33 @@ from vllm.models.minimax_m3.common.sparse_attention import (
|
||||
from vllm.models.minimax_m3.nvidia.msa_cutlass_sparse_decode import (
|
||||
MSACutlassDecodeMetadata,
|
||||
MSACutlassDecodePlanCache,
|
||||
is_nvfp4_kv_cache,
|
||||
msa_cutlass_sparse_decode,
|
||||
nvfp4_kv_cache_views,
|
||||
prepare_decode_metadata,
|
||||
should_prepare_decode_metadata,
|
||||
supports_cutlass_sparse_decode,
|
||||
)
|
||||
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionLayer,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _dequantize_query(
|
||||
query_fp8: torch.Tensor, q_scale: float, out: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Write ``query_fp8 * q_scale`` into ``out`` (E4M3 is exact in BF16)."""
|
||||
out.copy_(query_fp8)
|
||||
if q_scale != 1.0:
|
||||
out.mul_(q_scale)
|
||||
return out
|
||||
|
||||
|
||||
class MiniMaxM3SparseMSABackend(MiniMaxM3SparseBackend):
|
||||
"""MiniMax M3 backend with NVIDIA MSA-specific decode metadata."""
|
||||
|
||||
@@ -51,6 +67,30 @@ class MiniMaxM3SparseMSABackend(MiniMaxM3SparseBackend):
|
||||
return MiniMaxM3SparseMSAMetadataBuilder
|
||||
|
||||
|
||||
class MiniMaxM3SparseMSANvfp4Backend(MiniMaxM3SparseMSABackend):
|
||||
"""MSA backend over HND NVFP4 KV cache pages in per-head K/V slots."""
|
||||
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["nvfp4"]
|
||||
|
||||
@classmethod
|
||||
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
|
||||
"""Store each head's K and V as two slots of packed data + block scales
|
||||
(see ``nvfp4_kv_cache_views``)."""
|
||||
if spec.state_content_bytes is not None or not spec.kv_quant_mode.is_nvfp4:
|
||||
return spec
|
||||
return replace(
|
||||
spec,
|
||||
num_head_slots=2 * spec.num_kv_heads,
|
||||
state_content_bytes=nvfp4_kv_cache_full_dim(spec.head_size)
|
||||
* get_dtype_size(spec.dtype),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...] | None:
|
||||
# Every head's K/V slots are one contiguous run of the page.
|
||||
return (KVCacheLayout.LBHNC,)
|
||||
|
||||
|
||||
class MiniMaxM3SparseCutlassBackend(MiniMaxM3SparseMSABackend):
|
||||
"""Attention-backend alias selecting CUTLASS MSA sparse decode."""
|
||||
|
||||
@@ -166,6 +206,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
topk_blocks=topk_blocks,
|
||||
sparse_block_size=sparse_block_size,
|
||||
)
|
||||
self.use_nvfp4_kv = is_nvfp4_kv_cache(kv_cache_dtype)
|
||||
self.use_cutlass_decode = supports_cutlass_sparse_decode(
|
||||
decode_backend=msa_decode_backend,
|
||||
num_q_heads=self.num_heads,
|
||||
@@ -174,6 +215,12 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
page_size=self.block_size,
|
||||
topk_blocks=self.topk_blocks,
|
||||
)
|
||||
if self.use_nvfp4_kv and not self.use_cutlass_decode:
|
||||
raise ValueError(
|
||||
"MiniMax M3 NVFP4 KV cache requires CUTLASS MSA sparse decode "
|
||||
f"(num_heads={self.num_heads}, num_kv_heads={self.num_kv_heads}, "
|
||||
f"block_size={self.block_size}, topk_blocks={self.topk_blocks})"
|
||||
)
|
||||
logger.info_once(
|
||||
"MiniMax M3 MSA sparse decode selected %s",
|
||||
"CUTLASS" if self.use_cutlass_decode else "Triton",
|
||||
@@ -218,9 +265,17 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
hd = self.head_size
|
||||
q = query[:num_tokens].view(-1, self.num_heads, hd)
|
||||
out = output[:num_tokens].view(-1, self.num_heads, hd)
|
||||
kv_cache = (
|
||||
kv_cache.view(self.kv_cache_fp8_dtype) if self.use_fp8_kv else kv_cache
|
||||
# Given query_fp8, it is the only q the fused insert wrote: CUTLASS
|
||||
# decode reads it directly, and the bf16-query kernels read it
|
||||
# dequantized into ``query``.
|
||||
q_fp8 = (
|
||||
None
|
||||
if query_fp8 is None
|
||||
else query_fp8[:num_tokens].view(-1, self.num_heads, hd)
|
||||
)
|
||||
q_scale_float = getattr(layer, "_q_scale_float", 1.0)
|
||||
if self.use_fp8_kv and not self.use_nvfp4_kv:
|
||||
kv_cache = kv_cache.view(self.kv_cache_fp8_dtype)
|
||||
k_scale = getattr(layer, "_k_scale", None) if self.use_fp8_kv else None
|
||||
v_scale = getattr(layer, "_v_scale", None) if self.use_fp8_kv else None
|
||||
|
||||
@@ -234,21 +289,31 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
else None
|
||||
)
|
||||
if self.use_cutlass_decode and msa_metadata is not None:
|
||||
assert query_fp8 is not None
|
||||
assert q_fp8 is not None
|
||||
msa_cutlass_sparse_decode(
|
||||
query_fp8[:nd].view(-1, self.num_heads, hd),
|
||||
q_fp8[:nd],
|
||||
kv_cache,
|
||||
topk[:nd],
|
||||
out[:nd],
|
||||
msa_metadata,
|
||||
scale=self.scale,
|
||||
q_scale_float=getattr(layer, "_q_scale_float", 1.0),
|
||||
q_scale_float=q_scale_float,
|
||||
k_scale_float=getattr(layer, "_k_scale_float", 1.0),
|
||||
v_scale_float=getattr(layer, "_v_scale_float", 1.0),
|
||||
k_scale=k_scale if self.use_nvfp4_kv else None,
|
||||
v_scale=v_scale if self.use_nvfp4_kv else None,
|
||||
)
|
||||
elif self.use_nvfp4_kv:
|
||||
raise RuntimeError(
|
||||
"MiniMax M3 NVFP4 KV cache has no Triton decode fallback; "
|
||||
f"CUTLASS decode was not planned for {main_md.num_decodes} "
|
||||
f"requests with query length {d.decode_query_len}"
|
||||
)
|
||||
else:
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q[:nd],
|
||||
q[:nd]
|
||||
if q_fp8 is None
|
||||
else _dequantize_query(q_fp8[:nd], q_scale_float, q[:nd]),
|
||||
kv_cache,
|
||||
topk[:nd].transpose(0, 1),
|
||||
d.block_table,
|
||||
@@ -266,6 +331,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
from vllm.third_party.fmha_sm100.sparse import (
|
||||
build_k2q_csr,
|
||||
sparse_atten_func,
|
||||
sparse_atten_nvfp4_kv_func,
|
||||
)
|
||||
|
||||
p = main_md.prefill
|
||||
@@ -273,8 +339,11 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
# [H, prefill, MK] transposed view; build_k2q_csr consumes the
|
||||
# strided view directly (topK stays innermost-contiguous).
|
||||
prefill_topk = topk[nd:num_tokens].transpose(0, 1)
|
||||
qp = q[nd:]
|
||||
k_cache, v_cache = kv_cache.split(self.head_size, dim=-1)
|
||||
qp = (
|
||||
q[nd:]
|
||||
if q_fp8 is None
|
||||
else _dequantize_query(q_fp8[nd:], q_scale_float, q[nd:])
|
||||
)
|
||||
k2q_row_ptr, k2q_q_indices, schedule = build_k2q_csr(
|
||||
prefill_topk,
|
||||
p.cu_seqlens_q,
|
||||
@@ -287,6 +356,34 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
qhead_per_kv=qp.shape[1] // self.num_kv_heads,
|
||||
return_schedule=True,
|
||||
)
|
||||
if self.use_nvfp4_kv:
|
||||
k_data, k_sf, v_data, v_sf = nvfp4_kv_cache_views(kv_cache)
|
||||
sparse_atten_nvfp4_kv_func(
|
||||
qp,
|
||||
k_data,
|
||||
v_data,
|
||||
k_sf,
|
||||
v_sf,
|
||||
k_scale,
|
||||
v_scale,
|
||||
k2q_row_ptr,
|
||||
k2q_q_indices,
|
||||
self.topk_blocks,
|
||||
cu_seqlens_q=p.cu_seqlens_q,
|
||||
cu_seqlens_k=p.cu_seqlens_k,
|
||||
max_seqlen_q=p.max_query_len,
|
||||
max_seqlen_k=p.max_seq_len,
|
||||
blk_kv=SPARSE_BLOCK_SIZE,
|
||||
causal=True,
|
||||
softmax_scale=self.scale,
|
||||
page_table=p.block_table,
|
||||
seqused_k=p.seq_lens,
|
||||
schedule=schedule,
|
||||
out=out[nd:],
|
||||
kv_layout="vllm",
|
||||
)
|
||||
return output
|
||||
k_cache, v_cache = kv_cache.split(self.head_size, dim=-1)
|
||||
sparse_atten_func(
|
||||
qp,
|
||||
k_cache,
|
||||
|
||||
Reference in New Issue
Block a user