[Attention] Add FLASH_ATTN_MLA_SPARSE backend for Hopper sparse MLA (#46189)

Signed-off-by: Dakai An <dakaian108@gmail.com>
This commit is contained in:
Dakai An
2026-06-24 09:33:10 +00:00
committed by GitHub
parent d7c1821b5a
commit dc0d318177
4 changed files with 292 additions and 0 deletions
+1
View File
@@ -226,6 +226,7 @@ MLA decode backends are selected using the standard
| `FLASHMLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x-10.x |
| `FLASHMLA_SPARSE` | bf16 | `auto`, `bfloat16`, `fp8_ds_mla` | 64 | 576 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x |
| `FLASH_ATTN_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x |
| `FLASH_ATTN_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16` | 64 | Any | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x |
| `ROCM_AITER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %1 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A |
| `ROCM_AITER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 1, 64 | Any | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | N/A |
| `ROCM_AITER_TRITON_MLA` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A |
+1
View File
@@ -138,6 +138,7 @@ def _get_backend_priorities(
AttentionBackendEnum.FLASHMLA,
AttentionBackendEnum.FLASHINFER_MLA,
AttentionBackendEnum.TRITON_MLA,
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
AttentionBackendEnum.FLASHMLA_SPARSE,
]
else:
@@ -0,0 +1,287 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any, ClassVar
import numpy as np
import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.platforms.interface import DeviceCapability
from vllm.utils.torch_utils import np_to_pinned_tensor
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionLayer,
AttentionMetadata,
AttentionMetadataBuilder,
CommonAttentionMetadata,
MultipleOf,
SparseMLAAttentionImpl,
)
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
from vllm.v1.attention.backends.mla.sparse_utils import (
triton_convert_req_index_to_global_index,
)
from vllm.v1.kv_cache_interface import AttentionSpec
from vllm.vllm_flash_attn.flash_attn_interface import flash_attn_varlen_func
class FlashAttnMLASparseBackend(AttentionBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"auto",
"float16",
"bfloat16",
]
@staticmethod
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [64]
@staticmethod
def get_name() -> str:
return "FLASH_ATTN_MLA_SPARSE"
@staticmethod
def get_builder_cls() -> type["FlashAttnMLASparseMetadataBuilder"]:
return FlashAttnMLASparseMetadataBuilder
@staticmethod
def get_impl_cls() -> type[SparseMLAAttentionImpl[Any]]:
return FlashAttnMLASparseImpl
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
return []
@classmethod
def is_mla(cls) -> bool:
return True
@classmethod
def is_sparse(cls) -> bool:
return True
@classmethod
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
return capability.major == 9
@classmethod
def supports_combination(
cls,
head_size: int,
dtype: torch.dtype,
kv_cache_dtype: CacheDType | None,
block_size: int | None,
use_mla: bool,
has_sink: bool,
use_sparse: bool,
use_mm_prefix: bool,
device_capability: DeviceCapability,
) -> str | None:
if kv_cache_dtype not in (None, "auto", "float16", "bfloat16"):
return (
"FlashAttention MLA Sparse currently supports only FP16/BF16 KV cache"
)
if not flash_attn_supports_mla():
return "FlashAttention MLA not supported on this device"
from vllm.config import get_current_vllm_config_or_none
vllm_config = get_current_vllm_config_or_none()
if vllm_config is not None and vllm_config.model_config is not None:
if vllm_config.parallel_config.decode_context_parallel_size > 1:
return "FlashAttention MLA Sparse does not support DCP for now"
hf_config = vllm_config.model_config.hf_config
if not hasattr(hf_config, "index_topk"):
return "FlashAttention MLA Sparse requires model with index_topk"
return None
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
return (num_blocks, block_size, head_size)
@dataclass
class FlashAttnMLASparseMetadata(AttentionMetadata):
num_reqs: int
max_query_len: int
max_seq_len: int
num_actual_tokens: int
query_start_loc: torch.Tensor
slot_mapping: torch.Tensor
block_table: torch.Tensor
req_id_per_token: torch.Tensor
block_size: int = 64
topk_tokens: int = 2048
class FlashAttnMLASparseMetadataBuilder(
AttentionMetadataBuilder[FlashAttnMLASparseMetadata]
):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
def __init__(
self,
kv_cache_spec: AttentionSpec,
layer_names: list[str],
vllm_config: VllmConfig,
device: torch.device,
) -> None:
self.vllm_config = vllm_config
self.layer_names = layer_names
self.kv_cache_spec = kv_cache_spec
self.model_config = vllm_config.model_config
self.device = device
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
self.topk_tokens = vllm_config.model_config.hf_config.index_topk
self.req_id_per_token_buffer = torch.empty(
(vllm_config.scheduler_config.max_num_batched_tokens,),
dtype=torch.int32,
device=device,
)
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> FlashAttnMLASparseMetadata:
cm = common_attn_metadata
num_tokens = cm.num_actual_tokens
starts = np.asarray(cm.query_start_loc_cpu, dtype=np.int32)
seg_lengths = np.diff(starts)
req_id_per_token = np.repeat(
np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths
)
self.req_id_per_token_buffer.fill_(0)
self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_(
np_to_pinned_tensor(req_id_per_token), non_blocking=True
)
return FlashAttnMLASparseMetadata(
num_reqs=cm.num_reqs,
max_query_len=cm.max_query_len,
max_seq_len=cm.max_seq_len,
num_actual_tokens=cm.num_actual_tokens,
query_start_loc=cm.query_start_loc,
slot_mapping=cm.slot_mapping,
block_table=cm.block_table_tensor,
req_id_per_token=self.req_id_per_token_buffer[:num_tokens],
block_size=self.kv_cache_spec.block_size,
topk_tokens=self.topk_tokens,
)
class FlashAttnMLASparseImpl(SparseMLAAttentionImpl[FlashAttnMLASparseMetadata]):
def __init__(
self,
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: int,
alibi_slopes: list[float] | None,
sliding_window: int | None,
kv_cache_dtype: str,
logits_soft_cap: float | None,
attn_type: str,
kv_sharing_target_layer_name: str | None,
topk_indices_buffer: torch.Tensor | None = None,
indexer: Any | None = None,
**mla_args: Any,
) -> None:
unsupported_features = [alibi_slopes, sliding_window, logits_soft_cap]
if any(unsupported_features):
raise NotImplementedError(
"FlashAttnMLASparseImpl does not support alibi, sliding window, "
"or logits soft cap."
)
if kv_cache_dtype not in ("auto", "float16", "bfloat16"):
raise NotImplementedError(
"FlashAttnMLASparseImpl currently supports only FP16/BF16 KV cache."
)
self.num_heads = num_heads
self.head_size = head_size
self.scale = float(scale)
self.num_kv_heads = num_kv_heads
self.kv_cache_dtype = kv_cache_dtype
self.kv_lora_rank: int = mla_args["kv_lora_rank"]
self.qk_rope_head_dim: int = mla_args["qk_rope_head_dim"]
self.topk_indices_buffer: torch.Tensor | None = (
indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer
)
assert self.topk_indices_buffer is not None, (
"Indexer or topk_indices_buffer required for sparse MLA"
)
self.supports_quant_query_input = False
self.dcp_world_size = -1
self.q_pad_num_heads = None
def forward_mqa(
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: FlashAttnMLASparseMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if not isinstance(q, tuple):
raise NotImplementedError(
"FlashAttnMLASparseImpl expects split (q_nope, q_rope) input."
)
q_nope, q_rope = q
num_actual_toks = q_rope.shape[0]
assert self.topk_indices_buffer is not None
topk_indices = self.topk_indices_buffer[:num_actual_toks]
topk_indices, valid_counts = triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token[:num_actual_toks],
attn_metadata.block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
NUM_TOPK_TOKENS=topk_indices.shape[1],
return_valid_counts=True,
)
cu_seqlens_q = torch.arange(
0, num_actual_toks + 1, dtype=torch.int32, device=q_rope.device
)
kv_cache = kv_c_and_k_pe_cache.view(
-1, attn_metadata.block_size, self.head_size
)
k_cache = kv_cache[:, :, self.kv_lora_rank :].view(
-1, 1, 1, self.qk_rope_head_dim
)
v_cache = kv_cache[:, :, : self.kv_lora_rank].view(-1, 1, 1, self.kv_lora_rank)
out = flash_attn_varlen_func(
q=q_rope,
k=k_cache,
v=v_cache,
q_v=q_nope,
max_seqlen_q=1,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_k=topk_indices.shape[1],
seqused_k=valid_counts,
block_table=topk_indices,
softmax_scale=self.scale,
causal=True,
fa_version=3,
)
return out, None
+3
View File
@@ -95,6 +95,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
"vllm.models.deepseek_v4.amd.rocm.DeepseekV4ROCMAiterMLASparseBackend"
)
FLASH_ATTN_MLA = "vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend"
FLASH_ATTN_MLA_SPARSE = (
"vllm.v1.attention.backends.mla.flashattn_mla_sparse.FlashAttnMLASparseBackend"
)
MINIMAX_M3_SPARSE = (
"vllm.models.minimax_m3.common.sparse_attention.MiniMaxM3SparseBackend"
)