From dc0d318177e1086d483bff12ad71f1d3b5b980e7 Mon Sep 17 00:00:00 2001 From: Dakai An <77474977+andakai@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:33:10 +0800 Subject: [PATCH] [Attention] Add FLASH_ATTN_MLA_SPARSE backend for Hopper sparse MLA (#46189) Signed-off-by: Dakai An --- docs/design/attention_backends.md | 1 + vllm/platforms/cuda.py | 1 + .../backends/mla/flashattn_mla_sparse.py | 287 ++++++++++++++++++ vllm/v1/attention/backends/registry.py | 3 + 4 files changed, 292 insertions(+) create mode 100644 vllm/v1/attention/backends/mla/flashattn_mla_sparse.py diff --git a/docs/design/attention_backends.md b/docs/design/attention_backends.md index 4fee50068e..a8d2439f82 100644 --- a/docs/design/attention_backends.md +++ b/docs/design/attention_backends.md @@ -226,6 +226,7 @@ MLA decode backends are selected using the standard | `FLASHMLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x-10.x | | `FLASHMLA_SPARSE` | bf16 | `auto`, `bfloat16`, `fp8_ds_mla` | 64 | 576 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x | | `FLASH_ATTN_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x | +| `FLASH_ATTN_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16` | 64 | Any | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x | | `ROCM_AITER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %1 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A | | `ROCM_AITER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 1, 64 | Any | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | N/A | | `ROCM_AITER_TRITON_MLA` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A | diff --git a/vllm/platforms/cuda.py b/vllm/platforms/cuda.py index dabe6058e4..fa96cb8c94 100644 --- a/vllm/platforms/cuda.py +++ b/vllm/platforms/cuda.py @@ -138,6 +138,7 @@ def _get_backend_priorities( AttentionBackendEnum.FLASHMLA, AttentionBackendEnum.FLASHINFER_MLA, AttentionBackendEnum.TRITON_MLA, + AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE, AttentionBackendEnum.FLASHMLA_SPARSE, ] else: diff --git a/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py b/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py new file mode 100644 index 0000000000..664bd649fd --- /dev/null +++ b/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py @@ -0,0 +1,287 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from dataclasses import dataclass +from typing import Any, ClassVar + +import numpy as np +import torch + +from vllm.config import VllmConfig +from vllm.config.cache import CacheDType +from vllm.platforms.interface import DeviceCapability +from vllm.utils.torch_utils import np_to_pinned_tensor +from vllm.v1.attention.backend import ( + AttentionBackend, + AttentionCGSupport, + AttentionLayer, + AttentionMetadata, + AttentionMetadataBuilder, + CommonAttentionMetadata, + MultipleOf, + SparseMLAAttentionImpl, +) +from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla +from vllm.v1.attention.backends.mla.sparse_utils import ( + triton_convert_req_index_to_global_index, +) +from vllm.v1.kv_cache_interface import AttentionSpec +from vllm.vllm_flash_attn.flash_attn_interface import flash_attn_varlen_func + + +class FlashAttnMLASparseBackend(AttentionBackend): + supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16] + supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [ + "auto", + "float16", + "bfloat16", + ] + + @staticmethod + def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + return [64] + + @staticmethod + def get_name() -> str: + return "FLASH_ATTN_MLA_SPARSE" + + @staticmethod + def get_builder_cls() -> type["FlashAttnMLASparseMetadataBuilder"]: + return FlashAttnMLASparseMetadataBuilder + + @staticmethod + def get_impl_cls() -> type[SparseMLAAttentionImpl[Any]]: + return FlashAttnMLASparseImpl + + @classmethod + def get_supported_head_sizes(cls) -> list[int]: + return [] + + @classmethod + def is_mla(cls) -> bool: + return True + + @classmethod + def is_sparse(cls) -> bool: + return True + + @classmethod + def supports_compute_capability(cls, capability: DeviceCapability) -> bool: + return capability.major == 9 + + @classmethod + def supports_combination( + cls, + head_size: int, + dtype: torch.dtype, + kv_cache_dtype: CacheDType | None, + block_size: int | None, + use_mla: bool, + has_sink: bool, + use_sparse: bool, + use_mm_prefix: bool, + device_capability: DeviceCapability, + ) -> str | None: + if kv_cache_dtype not in (None, "auto", "float16", "bfloat16"): + return ( + "FlashAttention MLA Sparse currently supports only FP16/BF16 KV cache" + ) + + if not flash_attn_supports_mla(): + return "FlashAttention MLA not supported on this device" + + from vllm.config import get_current_vllm_config_or_none + + vllm_config = get_current_vllm_config_or_none() + if vllm_config is not None and vllm_config.model_config is not None: + if vllm_config.parallel_config.decode_context_parallel_size > 1: + return "FlashAttention MLA Sparse does not support DCP for now" + + hf_config = vllm_config.model_config.hf_config + if not hasattr(hf_config, "index_topk"): + return "FlashAttention MLA Sparse requires model with index_topk" + return None + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + cache_dtype_str: str = "auto", + ) -> tuple[int, ...]: + return (num_blocks, block_size, head_size) + + +@dataclass +class FlashAttnMLASparseMetadata(AttentionMetadata): + num_reqs: int + max_query_len: int + max_seq_len: int + + num_actual_tokens: int + query_start_loc: torch.Tensor + slot_mapping: torch.Tensor + + block_table: torch.Tensor + req_id_per_token: torch.Tensor + block_size: int = 64 + topk_tokens: int = 2048 + + +class FlashAttnMLASparseMetadataBuilder( + AttentionMetadataBuilder[FlashAttnMLASparseMetadata] +): + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + + def __init__( + self, + kv_cache_spec: AttentionSpec, + layer_names: list[str], + vllm_config: VllmConfig, + device: torch.device, + ) -> None: + self.vllm_config = vllm_config + self.layer_names = layer_names + self.kv_cache_spec = kv_cache_spec + self.model_config = vllm_config.model_config + self.device = device + + self._init_reorder_batch_threshold(1, supports_spec_as_decode=True) + + self.topk_tokens = vllm_config.model_config.hf_config.index_topk + self.req_id_per_token_buffer = torch.empty( + (vllm_config.scheduler_config.max_num_batched_tokens,), + dtype=torch.int32, + device=device, + ) + + def build( + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool = False, + ) -> FlashAttnMLASparseMetadata: + cm = common_attn_metadata + num_tokens = cm.num_actual_tokens + starts = np.asarray(cm.query_start_loc_cpu, dtype=np.int32) + seg_lengths = np.diff(starts) + req_id_per_token = np.repeat( + np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths + ) + + self.req_id_per_token_buffer.fill_(0) + self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_( + np_to_pinned_tensor(req_id_per_token), non_blocking=True + ) + + return FlashAttnMLASparseMetadata( + num_reqs=cm.num_reqs, + max_query_len=cm.max_query_len, + max_seq_len=cm.max_seq_len, + num_actual_tokens=cm.num_actual_tokens, + query_start_loc=cm.query_start_loc, + slot_mapping=cm.slot_mapping, + block_table=cm.block_table_tensor, + req_id_per_token=self.req_id_per_token_buffer[:num_tokens], + block_size=self.kv_cache_spec.block_size, + topk_tokens=self.topk_tokens, + ) + + +class FlashAttnMLASparseImpl(SparseMLAAttentionImpl[FlashAttnMLASparseMetadata]): + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: list[float] | None, + sliding_window: int | None, + kv_cache_dtype: str, + logits_soft_cap: float | None, + attn_type: str, + kv_sharing_target_layer_name: str | None, + topk_indices_buffer: torch.Tensor | None = None, + indexer: Any | None = None, + **mla_args: Any, + ) -> None: + unsupported_features = [alibi_slopes, sliding_window, logits_soft_cap] + if any(unsupported_features): + raise NotImplementedError( + "FlashAttnMLASparseImpl does not support alibi, sliding window, " + "or logits soft cap." + ) + if kv_cache_dtype not in ("auto", "float16", "bfloat16"): + raise NotImplementedError( + "FlashAttnMLASparseImpl currently supports only FP16/BF16 KV cache." + ) + + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.num_kv_heads = num_kv_heads + self.kv_cache_dtype = kv_cache_dtype + self.kv_lora_rank: int = mla_args["kv_lora_rank"] + self.qk_rope_head_dim: int = mla_args["qk_rope_head_dim"] + self.topk_indices_buffer: torch.Tensor | None = ( + indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer + ) + assert self.topk_indices_buffer is not None, ( + "Indexer or topk_indices_buffer required for sparse MLA" + ) + self.supports_quant_query_input = False + self.dcp_world_size = -1 + self.q_pad_num_heads = None + + def forward_mqa( + self, + q: torch.Tensor | tuple[torch.Tensor, torch.Tensor], + kv_c_and_k_pe_cache: torch.Tensor, + attn_metadata: FlashAttnMLASparseMetadata, + layer: AttentionLayer, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + if not isinstance(q, tuple): + raise NotImplementedError( + "FlashAttnMLASparseImpl expects split (q_nope, q_rope) input." + ) + q_nope, q_rope = q + num_actual_toks = q_rope.shape[0] + + assert self.topk_indices_buffer is not None + topk_indices = self.topk_indices_buffer[:num_actual_toks] + topk_indices, valid_counts = triton_convert_req_index_to_global_index( + attn_metadata.req_id_per_token[:num_actual_toks], + attn_metadata.block_table, + topk_indices, + BLOCK_SIZE=attn_metadata.block_size, + NUM_TOPK_TOKENS=topk_indices.shape[1], + return_valid_counts=True, + ) + + cu_seqlens_q = torch.arange( + 0, num_actual_toks + 1, dtype=torch.int32, device=q_rope.device + ) + kv_cache = kv_c_and_k_pe_cache.view( + -1, attn_metadata.block_size, self.head_size + ) + k_cache = kv_cache[:, :, self.kv_lora_rank :].view( + -1, 1, 1, self.qk_rope_head_dim + ) + v_cache = kv_cache[:, :, : self.kv_lora_rank].view(-1, 1, 1, self.kv_lora_rank) + + out = flash_attn_varlen_func( + q=q_rope, + k=k_cache, + v=v_cache, + q_v=q_nope, + max_seqlen_q=1, + cu_seqlens_q=cu_seqlens_q, + max_seqlen_k=topk_indices.shape[1], + seqused_k=valid_counts, + block_table=topk_indices, + softmax_scale=self.scale, + causal=True, + fa_version=3, + ) + return out, None diff --git a/vllm/v1/attention/backends/registry.py b/vllm/v1/attention/backends/registry.py index 5bbabc13dd..5fba5472f1 100644 --- a/vllm/v1/attention/backends/registry.py +++ b/vllm/v1/attention/backends/registry.py @@ -95,6 +95,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta): "vllm.models.deepseek_v4.amd.rocm.DeepseekV4ROCMAiterMLASparseBackend" ) FLASH_ATTN_MLA = "vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend" + FLASH_ATTN_MLA_SPARSE = ( + "vllm.v1.attention.backends.mla.flashattn_mla_sparse.FlashAttnMLASparseBackend" + ) MINIMAX_M3_SPARSE = ( "vllm.models.minimax_m3.common.sparse_attention.MiniMaxM3SparseBackend" )