Signed-off-by: Simon Veitner <sveitner@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
423 lines
15 KiB
Python
423 lines
15 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
import math
|
|
from dataclasses import dataclass
|
|
from functools import cache
|
|
from typing import Any, ClassVar
|
|
|
|
import torch
|
|
|
|
from vllm.config import VllmConfig
|
|
from vllm.config.cache import CacheDType
|
|
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
|
SparseMLACommonImpl,
|
|
SparseMLACommonMetadata,
|
|
SparseMLACommonMetadataBuilder,
|
|
)
|
|
from vllm.platforms.interface import DeviceCapability
|
|
from vllm.v1.attention.backend import (
|
|
AttentionBackend,
|
|
AttentionCGSupport,
|
|
AttentionLayer,
|
|
MLAAttentionImpl,
|
|
MultipleOf,
|
|
)
|
|
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
|
|
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
|
FlashInferMLASparseImpl,
|
|
FlashInferMLASparseTRTLLMBackend,
|
|
FlashInferMLASparseTRTLLMMetadataBuilder,
|
|
)
|
|
from vllm.v1.attention.backends.mla.sparse_utils import (
|
|
align_blocks_to_rows,
|
|
flat_kv_row_view,
|
|
)
|
|
from vllm.v1.attention.ops.metadata import compute_token_to_req_indices
|
|
from vllm.v1.kv_cache_interface import AttentionSpec
|
|
from vllm.vllm_flash_attn.flash_attn_interface import flash_attn_varlen_func
|
|
|
|
|
|
class FlashAttnMLASparseBackend(AttentionBackend):
|
|
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
|
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
|
"auto",
|
|
"float16",
|
|
"bfloat16",
|
|
]
|
|
|
|
@staticmethod
|
|
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
|
return [MultipleOf(64)]
|
|
|
|
@classmethod
|
|
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
|
|
return align_blocks_to_rows(spec)
|
|
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "FLASH_ATTN_MLA_SPARSE"
|
|
|
|
@staticmethod
|
|
def get_builder_cls() -> type["FlashAttnMLASparseMetadataBuilder"]:
|
|
return FlashAttnMLASparseMetadataBuilder
|
|
|
|
@staticmethod
|
|
def get_impl_cls() -> type[MLAAttentionImpl[Any]]:
|
|
return FlashAttnMLASparseImpl
|
|
|
|
@classmethod
|
|
def get_supported_head_sizes(cls) -> list[int]:
|
|
return []
|
|
|
|
@classmethod
|
|
def is_mla(cls) -> bool:
|
|
return True
|
|
|
|
@classmethod
|
|
def is_sparse(cls) -> bool:
|
|
return True
|
|
|
|
@classmethod
|
|
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
|
return capability.major == 9
|
|
|
|
@classmethod
|
|
def supports_combination(
|
|
cls,
|
|
head_size: int,
|
|
dtype: torch.dtype,
|
|
kv_cache_dtype: CacheDType | None,
|
|
block_size: int | None,
|
|
use_mla: bool,
|
|
has_sink: bool,
|
|
use_sparse: bool,
|
|
use_mm_prefix: bool,
|
|
device_capability: DeviceCapability,
|
|
) -> str | None:
|
|
if kv_cache_dtype not in (None, "auto", "float16", "bfloat16"):
|
|
return (
|
|
"FlashAttention MLA Sparse currently supports only FP16/BF16 KV cache"
|
|
)
|
|
|
|
if not flash_attn_supports_mla():
|
|
return "FlashAttention MLA not supported on this device"
|
|
|
|
from vllm.config import get_current_vllm_config_or_none
|
|
|
|
vllm_config = get_current_vllm_config_or_none()
|
|
if vllm_config is not None and vllm_config.model_config is not None:
|
|
if vllm_config.parallel_config.decode_context_parallel_size > 1:
|
|
return "FlashAttention MLA Sparse does not support DCP for now"
|
|
|
|
hf_text_config = vllm_config.model_config.hf_text_config
|
|
if not hasattr(hf_text_config, "index_topk"):
|
|
return "FlashAttention MLA Sparse requires model with index_topk"
|
|
return None
|
|
|
|
|
|
@dataclass
|
|
class FlashAttnMLASparseMetadata(SparseMLACommonMetadata):
|
|
pass
|
|
|
|
|
|
class FlashAttnMLASparseMetadataBuilder(
|
|
SparseMLACommonMetadataBuilder[FlashAttnMLASparseMetadata]
|
|
):
|
|
metadata_cls = FlashAttnMLASparseMetadata
|
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
|
|
|
def __init__(
|
|
self,
|
|
kv_cache_spec: AttentionSpec,
|
|
layer_names: list[str],
|
|
vllm_config: VllmConfig,
|
|
device: torch.device,
|
|
) -> None:
|
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
|
|
|
num_q_heads = self.model_config.get_num_attention_heads(
|
|
vllm_config.parallel_config
|
|
)
|
|
threshold = {16: 128, 32: 128, 64: 256, 128: 256}.get(num_q_heads, 256)
|
|
self._init_reorder_batch_threshold(threshold, supports_spec_as_decode=True)
|
|
self.supports_draft_decode_metadata_update = self.dcp_world_size == 1
|
|
|
|
def update_draft_decode_metadata(self, metadata: SparseMLACommonMetadata) -> None:
|
|
num_tokens = metadata.num_decode_tokens
|
|
if num_tokens == 0:
|
|
return
|
|
# Everything else is a view of runner buffers; the per-token request
|
|
# map is a builder buffer that build() filled for the captured batch.
|
|
compute_token_to_req_indices(
|
|
metadata.query_start_loc,
|
|
metadata.req_id_per_token,
|
|
num_tokens,
|
|
num_tokens,
|
|
)
|
|
|
|
|
|
class FlashAttnMLASparseImpl(SparseMLACommonImpl[FlashAttnMLASparseMetadata]):
|
|
group_conversion_decode_only: ClassVar[bool] = True
|
|
|
|
def __init__(
|
|
self,
|
|
num_heads: int,
|
|
head_size: int,
|
|
scale: float,
|
|
num_kv_heads: int,
|
|
alibi_slopes: list[float] | None,
|
|
sliding_window: int | None,
|
|
kv_cache_dtype: str,
|
|
logits_soft_cap: float | None,
|
|
attn_type: str,
|
|
kv_sharing_target_layer_name: str | None,
|
|
topk_indices_buffer: torch.Tensor | None = None,
|
|
indexer: Any | None = None,
|
|
**mla_args: Any,
|
|
) -> None:
|
|
unsupported_features = [alibi_slopes, sliding_window, logits_soft_cap]
|
|
if any(unsupported_features):
|
|
raise NotImplementedError(
|
|
"FlashAttnMLASparseImpl does not support alibi, sliding window, "
|
|
"or logits soft cap."
|
|
)
|
|
if kv_cache_dtype not in ("auto", "float16", "bfloat16"):
|
|
raise NotImplementedError(
|
|
"FlashAttnMLASparseImpl currently supports only FP16/BF16 KV cache."
|
|
)
|
|
|
|
super().__init__(
|
|
num_heads,
|
|
head_size,
|
|
scale,
|
|
num_kv_heads,
|
|
alibi_slopes,
|
|
sliding_window,
|
|
kv_cache_dtype,
|
|
logits_soft_cap,
|
|
attn_type,
|
|
kv_sharing_target_layer_name,
|
|
indexer=indexer,
|
|
topk_indices_buffer=topk_indices_buffer,
|
|
**mla_args,
|
|
)
|
|
assert self.topk_indices_buffer is not None, (
|
|
"Indexer or topk_indices_buffer required for sparse MLA"
|
|
)
|
|
self.cu_seqlens_q_buffer = torch.arange(
|
|
self.topk_indices_buffer.shape[0] + 1,
|
|
dtype=torch.int32,
|
|
device=self.topk_indices_buffer.device,
|
|
)
|
|
self.supports_quant_query_input = False
|
|
|
|
def _forward_mqa_kernel(
|
|
self,
|
|
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
|
kv_cache: torch.Tensor,
|
|
topk_indices: torch.Tensor,
|
|
valid_counts: torch.Tensor,
|
|
*,
|
|
layer: AttentionLayer,
|
|
block_size: int,
|
|
is_decode: bool,
|
|
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
|
if not isinstance(q, tuple):
|
|
raise NotImplementedError(
|
|
"FlashAttnMLASparseImpl expects split (q_nope, q_rope) input."
|
|
)
|
|
q_nope, q_rope = q
|
|
kv_rows, _ = flat_kv_row_view(kv_cache, block_size)
|
|
|
|
cu_seqlens_q = self.cu_seqlens_q_buffer[: q_rope.shape[0] + 1]
|
|
v_cache = kv_rows[:, : self.kv_lora_rank].unsqueeze(1).unsqueeze(1)
|
|
if self.qk_rope_head_dim == 0:
|
|
# FA3's QV path requires the 64-wide Q/K specialization. For NoPE
|
|
# MLA, zero Q preserves QV-only attention scores while providing a
|
|
# valid TMA shape.
|
|
_FA3_QV_HEAD_DIM = 64
|
|
q_rope = q_nope.new_zeros(*q_nope.shape[:-1], _FA3_QV_HEAD_DIM)
|
|
k_cache = v_cache[..., :_FA3_QV_HEAD_DIM]
|
|
else:
|
|
k_cache = kv_rows[:, self.kv_lora_rank :].unsqueeze(1).unsqueeze(1)
|
|
|
|
out = flash_attn_varlen_func(
|
|
q=q_rope,
|
|
k=k_cache,
|
|
v=v_cache,
|
|
q_v=q_nope,
|
|
max_seqlen_q=1,
|
|
cu_seqlens_q=cu_seqlens_q,
|
|
max_seqlen_k=topk_indices.shape[1],
|
|
seqused_k=valid_counts,
|
|
block_table=topk_indices,
|
|
softmax_scale=self.scale,
|
|
causal=True,
|
|
fa_version=3,
|
|
)
|
|
return out, None
|
|
|
|
|
|
def _fa4_cute_mla_available() -> str | None:
|
|
"""Reason the FA4 cute-DSL MLA entry point is unusable, or None."""
|
|
from vllm.vllm_flash_attn import flash_attn_interface as fa
|
|
|
|
if not fa.is_fa_version_supported(4):
|
|
return f"FA4 unavailable: {fa.fa_version_unsupported_reason(4)}"
|
|
try:
|
|
from vllm.vllm_flash_attn.cute.interface import _flash_attn_fwd # noqa: F401
|
|
except Exception as e: # cute-DSL / cutlass import chain
|
|
return f"FA4 cute-DSL entry point failed to import: {e!r}"
|
|
return None
|
|
|
|
|
|
class FlashAttnMLASparseFA4Backend(FlashInferMLASparseTRTLLMBackend):
|
|
"""FA4 for is_decode kernel calls; trtllm-gen for the rest."""
|
|
|
|
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
|
# The qv kernel asserts every descale is None: BF16 KV cache only.
|
|
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"]
|
|
|
|
@staticmethod
|
|
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
|
return [MultipleOf(64)]
|
|
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "FLASH_ATTN_MLA_SPARSE_FA4"
|
|
|
|
@staticmethod
|
|
def get_builder_cls() -> type["FlashAttnMLASparseFA4MetadataBuilder"]:
|
|
return FlashAttnMLASparseFA4MetadataBuilder
|
|
|
|
@staticmethod
|
|
def get_impl_cls() -> type[MLAAttentionImpl[Any]]:
|
|
return FlashAttnMLASparseFA4Impl
|
|
|
|
@classmethod
|
|
def get_supported_head_sizes(cls) -> list[int]:
|
|
# 512 latent + 64 RoPE; split back apart for the qv kernel.
|
|
return [576]
|
|
|
|
@classmethod
|
|
def supports_combination(cls, *args: Any, **kwargs: Any) -> str | None:
|
|
from vllm.config import get_current_vllm_config_or_none
|
|
from vllm.utils.flashinfer import has_flashinfer
|
|
|
|
if not has_flashinfer():
|
|
return "FA4 sparse MLA runs its prefill batches on FlashInfer's kernel"
|
|
if (reason := _fa4_cute_mla_available()) is not None:
|
|
return reason
|
|
vllm_config = get_current_vllm_config_or_none()
|
|
if vllm_config is None:
|
|
return None
|
|
if vllm_config.model_config is not None:
|
|
hf_config = vllm_config.model_config.hf_text_config
|
|
topk = getattr(hf_config, "index_topk", None)
|
|
# gather_kv_indices' last dim must be a whole number of n-tiles.
|
|
if topk is None or topk % 128 != 0:
|
|
return f"FA4 sparse MLA requires index_topk % 128 == 0, got {topk}"
|
|
dims = (
|
|
getattr(hf_config, "kv_lora_rank", None),
|
|
getattr(hf_config, "qk_rope_head_dim", None),
|
|
)
|
|
if dims != (512, 64):
|
|
return (
|
|
"FA4 sparse MLA requires (kv_lora_rank, qk_rope_head_dim) "
|
|
f"== (512, 64), got {dims}"
|
|
)
|
|
# Under DCP the query is all-gathered: the kernel sees heads * dcp_size.
|
|
dcp_size = vllm_config.parallel_config.decode_context_parallel_size
|
|
num_heads = vllm_config.model_config.get_num_attention_heads(
|
|
vllm_config.parallel_config
|
|
)
|
|
# 128 gathered heads take FA's prefill kernel, unvalidated under DCP.
|
|
head_counts = (8, 16, 32, 64) if dcp_size > 1 else (8, 16, 32, 64, 128)
|
|
if num_heads * dcp_size not in head_counts:
|
|
return (
|
|
f"FA4 sparse MLA requires {head_counts} gathered query heads, "
|
|
f"got num_heads={num_heads} * dcp_size={dcp_size}"
|
|
)
|
|
return super().supports_combination(*args, **kwargs)
|
|
|
|
|
|
class FlashAttnMLASparseFA4MetadataBuilder(FlashInferMLASparseTRTLLMMetadataBuilder):
|
|
"""The trtllm-gen builder, for the prefill lane's DCP workspace pre-size.
|
|
|
|
Its UNIFORM_BATCH support and varlen decode bound also fit FA4, which runs
|
|
one query row per token and bakes the row count into its varlen scalars.
|
|
"""
|
|
|
|
|
|
@cache
|
|
def _fa4_varlen_scalars(
|
|
num_tokens: int, num_kv_rows: int, device: torch.device
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Per-token varlen scalars; never evict, CUDA graphs bake in the pointers."""
|
|
cu_seqlens_q = torch.arange(num_tokens + 1, dtype=torch.int32, device=device)
|
|
cu_seqlens_k = torch.zeros(num_tokens + 1, dtype=torch.int32, device=device)
|
|
seqused_k = torch.full((num_tokens,), num_kv_rows, dtype=torch.int32, device=device)
|
|
return cu_seqlens_q, cu_seqlens_k, seqused_k
|
|
|
|
|
|
class FlashAttnMLASparseFA4Impl(FlashInferMLASparseImpl):
|
|
# FA4 emits a natural-log LSE; prefill converts trtllm-gen's log2 LSE.
|
|
lse_base_on_e: bool = True
|
|
|
|
def _forward_mqa_kernel(
|
|
self,
|
|
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
|
kv_cache: torch.Tensor,
|
|
topk_indices: torch.Tensor,
|
|
valid_counts: torch.Tensor,
|
|
*,
|
|
layer: AttentionLayer,
|
|
block_size: int,
|
|
is_decode: bool,
|
|
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
|
if not is_decode:
|
|
out, lse = super()._forward_mqa_kernel(
|
|
q,
|
|
kv_cache,
|
|
topk_indices,
|
|
valid_counts,
|
|
layer=layer,
|
|
block_size=block_size,
|
|
is_decode=is_decode,
|
|
)
|
|
return out, None if lse is None else lse * math.log(2.0)
|
|
if isinstance(q, tuple):
|
|
ql_nope, q_pe = q
|
|
else:
|
|
# 16B-aligned halves of the fused query; FA4 reads them in place.
|
|
ql_nope, q_pe = q[..., : self.kv_lora_rank], q[..., self.kv_lora_rank :]
|
|
kv_rows, _ = flat_kv_row_view(kv_cache, block_size)
|
|
num_tokens = q_pe.shape[0]
|
|
num_kv_rows = kv_rows.shape[0]
|
|
assert self.topk_indices_buffer is not None
|
|
cu_seqlens_q, cu_seqlens_k, seqused_k = _fa4_varlen_scalars(
|
|
self.topk_indices_buffer.shape[0], num_kv_rows, kv_rows.device
|
|
)
|
|
# An all-sentinel row yields (0, -inf), the DCP merge identity; no masking.
|
|
kernel_out = flash_attn_varlen_func(
|
|
q=q_pe,
|
|
k=kv_rows[:, self.kv_lora_rank :].unsqueeze(1),
|
|
v=kv_rows[:, : self.kv_lora_rank].unsqueeze(1),
|
|
q_v=ql_nope,
|
|
max_seqlen_q=1,
|
|
cu_seqlens_q=cu_seqlens_q[: num_tokens + 1],
|
|
max_seqlen_k=num_kv_rows,
|
|
cu_seqlens_k=cu_seqlens_k[: num_tokens + 1],
|
|
seqused_k=seqused_k[:num_tokens],
|
|
gather_kv_indices=topk_indices,
|
|
# Part of FA4's compile key: pass it unconditionally.
|
|
gather_kv_valid_length=valid_counts,
|
|
softmax_scale=self.scale,
|
|
# Causality is in the top-k list; causal=True would clamp flat rows.
|
|
causal=False,
|
|
fa_version=4,
|
|
return_softmax_lse=self.need_to_return_lse_for_decode,
|
|
)
|
|
return kernel_out if self.need_to_return_lse_for_decode else (kernel_out, None)
|