[Perf][Attention][MiniMax-M3] Select CUTLASS MSA by EAGLE3 decode shape (#55838)

Signed-off-by: Anish Shanbhag <ashanbhag@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Anish Shanbhag
2026-10-13 15:05:28 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent 6d1daa9fe5
commit 3f550129e8
3 changed files with 62 additions and 17 deletions
@@ -23,9 +23,10 @@ _TOPK = 16
# fixed planner allocation used by the MSA decode kernel.
_MAX_QUERY_HEAD_ROWS = 65536
_MAX_DECODE_QUERY_LEN = 32
# Kernel benchmarks put the CUTLASS crossover at 16 requests for TP1 and TP4.
# NVFP4 has no Triton fallback, so it takes CUTLASS at every batch size.
_MIN_CUTLASS_BATCH_SIZE = 16
_DEFAULT_MIN_CUTLASS_BATCH_SIZE = 16
_B300_EAGLE3_QUERY_HEAD_ROWS = 1024
_B300_EAGLE3_MAX_QUERY_LEN = 8
_B300_EAGLE3_HEAD_GEOMETRIES = {(64, 4), (32, 2), (16, 1)}
def is_nvfp4_kv_cache(kv_cache_dtype: str) -> bool:
@@ -235,6 +236,22 @@ def _supported_head_geometry(num_q_heads: int, num_kv_heads: int) -> bool:
)
def _min_cutlass_batch_size(
decode_query_len: int,
num_q_heads: int,
num_kv_heads: int,
) -> int:
if (
not current_platform.is_device_capability((10, 3))
or not 1 <= decode_query_len <= _B300_EAGLE3_MAX_QUERY_LEN
or (num_q_heads, num_kv_heads) not in _B300_EAGLE3_HEAD_GEOMETRIES
):
return _DEFAULT_MIN_CUTLASS_BATCH_SIZE
# Use CUTLASS once batch_size * decode_query_len * num_q_heads >= 1024.
query_head_rows = decode_query_len * num_q_heads
return (_B300_EAGLE3_QUERY_HEAD_ROWS + query_head_rows - 1) // query_head_rows
def supports_cutlass_sparse_decode(
*,
decode_backend: MiniMaxM3MSADecodeBackend,
@@ -267,9 +284,14 @@ def should_prepare_decode_metadata(
kv_cache_dtype: str,
page_size: int,
topk_blocks: int,
min_batch_size: int | None = None,
) -> bool:
"""Return whether a graph shape can use the CUTLASS decode path."""
total_q = batch_size * decode_query_len
if min_batch_size is None:
min_batch_size = _min_cutlass_batch_size(
decode_query_len, num_q_heads, num_kv_heads
)
return (
supports_cutlass_sparse_decode(
decode_backend=decode_backend,
@@ -280,7 +302,7 @@ def should_prepare_decode_metadata(
topk_blocks=topk_blocks,
)
and 1 <= decode_query_len <= _MAX_DECODE_QUERY_LEN
and (batch_size >= _MIN_CUTLASS_BATCH_SIZE or is_nvfp4_kv_cache(kv_cache_dtype))
and (batch_size >= min_batch_size or is_nvfp4_kv_cache(kv_cache_dtype))
and total_q * num_q_heads <= _MAX_QUERY_HEAD_ROWS
)
@@ -8,6 +8,7 @@ path for regular decode and speculative verification. NVFP4 KV caches read the
vLLM packed pages directly in both prefill and CUTLASS decode.
"""
import os
from dataclasses import dataclass, replace
from typing import ClassVar
@@ -49,6 +50,11 @@ from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
logger = init_logger(__name__)
def _get_msa_cutlass_min_batch_size() -> int | None:
value = os.getenv("VLLM_MINIMAX_M3_MSA_CUTLASS_MIN_BATCH")
return int(value) if value is not None else None
def _dequantize_query(
query_fp8: torch.Tensor, q_scale: float, out: torch.Tensor
) -> torch.Tensor:
@@ -133,6 +139,7 @@ class MiniMaxM3SparseMSAMetadataBuilder(MiniMaxM3SparseMetadataBuilder):
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
self.decode_backend = vllm_config.attention_config.minimax_m3_msa_decode_backend
self.msa_cutlass_plan_cache = MSACutlassDecodePlanCache()
self.msa_cutlass_min_batch_size = _get_msa_cutlass_min_batch_size()
def build(
self,
@@ -159,6 +166,7 @@ class MiniMaxM3SparseMSAMetadataBuilder(MiniMaxM3SparseMetadataBuilder):
kv_cache_dtype=self.kv_cache_dtype,
page_size=SPARSE_BLOCK_SIZE,
topk_blocks=self.topk_blocks,
min_batch_size=self.msa_cutlass_min_batch_size,
):
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
assert seq_lens_cpu is not None