[Perf][Attention][MiniMax-M3] Select CUTLASS MSA by EAGLE3 decode shape (#55838)
Signed-off-by: Anish Shanbhag <ashanbhag@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
6d1daa9fe5
commit
3f550129e8
@@ -23,9 +23,10 @@ _TOPK = 16
|
||||
# fixed planner allocation used by the MSA decode kernel.
|
||||
_MAX_QUERY_HEAD_ROWS = 65536
|
||||
_MAX_DECODE_QUERY_LEN = 32
|
||||
# Kernel benchmarks put the CUTLASS crossover at 16 requests for TP1 and TP4.
|
||||
# NVFP4 has no Triton fallback, so it takes CUTLASS at every batch size.
|
||||
_MIN_CUTLASS_BATCH_SIZE = 16
|
||||
_DEFAULT_MIN_CUTLASS_BATCH_SIZE = 16
|
||||
_B300_EAGLE3_QUERY_HEAD_ROWS = 1024
|
||||
_B300_EAGLE3_MAX_QUERY_LEN = 8
|
||||
_B300_EAGLE3_HEAD_GEOMETRIES = {(64, 4), (32, 2), (16, 1)}
|
||||
|
||||
|
||||
def is_nvfp4_kv_cache(kv_cache_dtype: str) -> bool:
|
||||
@@ -235,6 +236,22 @@ def _supported_head_geometry(num_q_heads: int, num_kv_heads: int) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _min_cutlass_batch_size(
|
||||
decode_query_len: int,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
) -> int:
|
||||
if (
|
||||
not current_platform.is_device_capability((10, 3))
|
||||
or not 1 <= decode_query_len <= _B300_EAGLE3_MAX_QUERY_LEN
|
||||
or (num_q_heads, num_kv_heads) not in _B300_EAGLE3_HEAD_GEOMETRIES
|
||||
):
|
||||
return _DEFAULT_MIN_CUTLASS_BATCH_SIZE
|
||||
# Use CUTLASS once batch_size * decode_query_len * num_q_heads >= 1024.
|
||||
query_head_rows = decode_query_len * num_q_heads
|
||||
return (_B300_EAGLE3_QUERY_HEAD_ROWS + query_head_rows - 1) // query_head_rows
|
||||
|
||||
|
||||
def supports_cutlass_sparse_decode(
|
||||
*,
|
||||
decode_backend: MiniMaxM3MSADecodeBackend,
|
||||
@@ -267,9 +284,14 @@ def should_prepare_decode_metadata(
|
||||
kv_cache_dtype: str,
|
||||
page_size: int,
|
||||
topk_blocks: int,
|
||||
min_batch_size: int | None = None,
|
||||
) -> bool:
|
||||
"""Return whether a graph shape can use the CUTLASS decode path."""
|
||||
total_q = batch_size * decode_query_len
|
||||
if min_batch_size is None:
|
||||
min_batch_size = _min_cutlass_batch_size(
|
||||
decode_query_len, num_q_heads, num_kv_heads
|
||||
)
|
||||
return (
|
||||
supports_cutlass_sparse_decode(
|
||||
decode_backend=decode_backend,
|
||||
@@ -280,7 +302,7 @@ def should_prepare_decode_metadata(
|
||||
topk_blocks=topk_blocks,
|
||||
)
|
||||
and 1 <= decode_query_len <= _MAX_DECODE_QUERY_LEN
|
||||
and (batch_size >= _MIN_CUTLASS_BATCH_SIZE or is_nvfp4_kv_cache(kv_cache_dtype))
|
||||
and (batch_size >= min_batch_size or is_nvfp4_kv_cache(kv_cache_dtype))
|
||||
and total_q * num_q_heads <= _MAX_QUERY_HEAD_ROWS
|
||||
)
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ path for regular decode and speculative verification. NVFP4 KV caches read the
|
||||
vLLM packed pages directly in both prefill and CUTLASS decode.
|
||||
"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import ClassVar
|
||||
|
||||
@@ -49,6 +50,11 @@ from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_msa_cutlass_min_batch_size() -> int | None:
|
||||
value = os.getenv("VLLM_MINIMAX_M3_MSA_CUTLASS_MIN_BATCH")
|
||||
return int(value) if value is not None else None
|
||||
|
||||
|
||||
def _dequantize_query(
|
||||
query_fp8: torch.Tensor, q_scale: float, out: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
@@ -133,6 +139,7 @@ class MiniMaxM3SparseMSAMetadataBuilder(MiniMaxM3SparseMetadataBuilder):
|
||||
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
|
||||
self.decode_backend = vllm_config.attention_config.minimax_m3_msa_decode_backend
|
||||
self.msa_cutlass_plan_cache = MSACutlassDecodePlanCache()
|
||||
self.msa_cutlass_min_batch_size = _get_msa_cutlass_min_batch_size()
|
||||
|
||||
def build(
|
||||
self,
|
||||
@@ -159,6 +166,7 @@ class MiniMaxM3SparseMSAMetadataBuilder(MiniMaxM3SparseMetadataBuilder):
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
page_size=SPARSE_BLOCK_SIZE,
|
||||
topk_blocks=self.topk_blocks,
|
||||
min_batch_size=self.msa_cutlass_min_batch_size,
|
||||
):
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
|
||||
Reference in New Issue
Block a user