702 lines
25 KiB
Python
702 lines
25 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""FlashInfer sparse MLA attention backend."""
|
|
|
|
import math
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING, ClassVar
|
|
|
|
import torch
|
|
|
|
from vllm import envs
|
|
from vllm.config import VllmConfig
|
|
from vllm.config.cache import CacheDType
|
|
from vllm.logger import init_logger
|
|
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
|
SparseMLACommonImpl,
|
|
SparseMLACommonMetadata,
|
|
SparseMLACommonMetadataBuilder,
|
|
)
|
|
from vllm.platforms import current_platform
|
|
from vllm.platforms.interface import DeviceCapability
|
|
from vllm.utils.torch_utils import is_quantized_kv_cache
|
|
from vllm.v1.attention.backend import (
|
|
AttentionBackend,
|
|
AttentionCGSupport,
|
|
AttentionLayer,
|
|
AttentionType,
|
|
MLAAttentionImpl,
|
|
MultipleOf,
|
|
max_decode_query_len,
|
|
)
|
|
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
|
from vllm.v1.attention.backends.mla.sparse_utils import (
|
|
align_blocks_to_rows,
|
|
flat_kv_row_view,
|
|
prepare_sparse_mla_safe_lengths,
|
|
)
|
|
from vllm.v1.attention.ops.metadata import compute_token_to_req_indices
|
|
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.model_executor.models.deepseek_v2 import Indexer
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class _FlashInferMLASparseBackendBase(AttentionBackend):
|
|
"""Common metadata for concrete FlashInfer sparse MLA backends."""
|
|
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "FLASHINFER_MLA_SPARSE"
|
|
|
|
@staticmethod
|
|
def get_builder_cls() -> type["FlashInferMLASparseMetadataBuilder"]:
|
|
return FlashInferMLASparseMetadataBuilder
|
|
|
|
@classmethod
|
|
def get_supported_head_sizes(cls) -> list[int]:
|
|
# 576 = 512 NoPE + 64 RoPE (with-rope layout); 512 = 512 NoPE only
|
|
# (no-rope layout, qk_rope_head_dim == 0). Both share D_V = 512 and are
|
|
# served by the TRTLLM-GEN sparse MLA kernel.
|
|
return [512, 576]
|
|
|
|
@classmethod
|
|
def is_mla(cls) -> bool:
|
|
return True
|
|
|
|
@classmethod
|
|
def is_sparse(cls) -> bool:
|
|
return True
|
|
|
|
@classmethod
|
|
def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec:
|
|
# The kernels read the rows as pages of their kernel block size.
|
|
(page,) = cls.get_supported_kernel_block_sizes(spec)
|
|
assert isinstance(page, MultipleOf)
|
|
return align_blocks_to_rows(spec, page.base)
|
|
|
|
|
|
class FlashInferMLASparseTRTLLMBackend(_FlashInferMLASparseBackendBase):
|
|
"""FlashInfer sparse MLA backend using the TRTLLM-gen launcher."""
|
|
|
|
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
|
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
|
"auto",
|
|
"float16",
|
|
"bfloat16",
|
|
"fp8",
|
|
"fp8_e4m3",
|
|
]
|
|
|
|
@staticmethod
|
|
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
|
return [MultipleOf(32)]
|
|
|
|
@staticmethod
|
|
def get_impl_cls() -> type[MLAAttentionImpl]:
|
|
return FlashInferMLASparseImpl
|
|
|
|
@staticmethod
|
|
def get_builder_cls() -> type["FlashInferMLASparseTRTLLMMetadataBuilder"]:
|
|
return FlashInferMLASparseTRTLLMMetadataBuilder
|
|
|
|
@classmethod
|
|
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
|
return capability.major == 10
|
|
|
|
@classmethod
|
|
def supports_combination(
|
|
cls,
|
|
head_size: int,
|
|
dtype: torch.dtype,
|
|
kv_cache_dtype: CacheDType | None,
|
|
block_size: int | None,
|
|
use_mla: bool,
|
|
has_sink: bool,
|
|
use_sparse: bool,
|
|
use_mm_prefix: bool,
|
|
device_capability: DeviceCapability,
|
|
) -> str | None:
|
|
from vllm.config import get_current_vllm_config
|
|
|
|
vllm_config = get_current_vllm_config()
|
|
parallel_config = vllm_config.parallel_config
|
|
if (
|
|
parallel_config.prefill_context_parallel_size > 1
|
|
and parallel_config.decode_context_parallel_size > 1
|
|
):
|
|
return (
|
|
f"{cls.get_name()} does not support combined PCP+DCP; "
|
|
"use FLASHMLA_SPARSE, which gathers each DCP KV shard before "
|
|
"running the rank-local PCP prefill queries"
|
|
)
|
|
if kv_cache_dtype == "fp8_ds_mla":
|
|
return f"{cls.get_name()} SM10 does not support fp8_ds_mla kv-cache dtype"
|
|
|
|
# FlashInfer MLA sparse SM10 kernel requires qk_nope_head_dim in [128, 192].
|
|
if vllm_config.model_config is not None:
|
|
hf_text_config = vllm_config.model_config.hf_text_config
|
|
qk_nope_head_dim = hf_text_config.qk_nope_head_dim
|
|
qk_rope_head_dim = hf_text_config.qk_rope_head_dim
|
|
kv_lora_rank = hf_text_config.kv_lora_rank
|
|
if qk_rope_head_dim == 0:
|
|
# Native no-rope MLA: FlashInfer only ships the one shape
|
|
# (nope_mla_dimensions in flashinfer.mla._core).
|
|
if qk_nope_head_dim != 256 or kv_lora_rank != 512:
|
|
return (
|
|
"FlashInfer native no-rope MLA requires "
|
|
"qk_nope_head_dim=256 and kv_lora_rank=512, but got "
|
|
f"qk_nope_head_dim={qk_nope_head_dim}, "
|
|
f"kv_lora_rank={kv_lora_rank}"
|
|
)
|
|
elif qk_nope_head_dim not in [128, 192]:
|
|
return (
|
|
"FlashInfer MLA Sparse kernel requires qk_nope_head_dim "
|
|
f"in [128, 192], but got {qk_nope_head_dim}"
|
|
)
|
|
# Check for index_topk which indicates sparse model
|
|
if not hasattr(hf_text_config, "index_topk"):
|
|
return "FlashInfer MLA Sparse requires model with index_topk config"
|
|
return None
|
|
|
|
|
|
class FlashInferMLASparseSM120Backend(_FlashInferMLASparseBackendBase):
|
|
"""FlashInfer sparse MLA backend for SM120."""
|
|
|
|
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
|
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
|
"auto",
|
|
"fp8",
|
|
"fp8_e4m3",
|
|
"fp8_ds_mla",
|
|
]
|
|
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "FLASHINFER_MLA_SPARSE_SM120"
|
|
|
|
@staticmethod
|
|
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
|
return [MultipleOf(64)]
|
|
|
|
@staticmethod
|
|
def get_impl_cls() -> type[MLAAttentionImpl]:
|
|
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm120 import (
|
|
FlashInferMLASparseSM120Impl,
|
|
)
|
|
|
|
return FlashInferMLASparseSM120Impl
|
|
|
|
@classmethod
|
|
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
|
return capability.major == 12
|
|
|
|
@classmethod
|
|
def supports_combination(
|
|
cls,
|
|
head_size: int,
|
|
dtype: torch.dtype,
|
|
kv_cache_dtype: CacheDType | None,
|
|
block_size: int | None,
|
|
use_mla: bool,
|
|
has_sink: bool,
|
|
use_sparse: bool,
|
|
use_mm_prefix: bool,
|
|
device_capability: DeviceCapability,
|
|
) -> str | None:
|
|
from vllm.config import get_current_vllm_config
|
|
from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120
|
|
|
|
if not has_flashinfer_sparse_mla_sm120():
|
|
return (
|
|
"FLASHINFER_MLA_SPARSE_SM120 requires FlashInfer's "
|
|
"sparse MLA decode API"
|
|
)
|
|
if dtype != torch.bfloat16:
|
|
return "dtype not supported"
|
|
if kv_cache_dtype not in (
|
|
None,
|
|
"auto",
|
|
"fp8",
|
|
"fp8_e4m3",
|
|
"fp8_ds_mla",
|
|
):
|
|
return "kv_cache_dtype not supported"
|
|
vllm_config = get_current_vllm_config()
|
|
if vllm_config.model_config is not None:
|
|
hf_text_config = vllm_config.model_config.hf_text_config
|
|
index_topk = getattr(hf_text_config, "index_topk", None)
|
|
if index_topk is None:
|
|
return (
|
|
"FLASHINFER_MLA_SPARSE_SM120 requires a model with "
|
|
"index_topk config"
|
|
)
|
|
if int(index_topk) != 2048:
|
|
return (
|
|
"FLASHINFER_MLA_SPARSE_SM120 requires index_topk=2048; "
|
|
f"got {index_topk}"
|
|
)
|
|
return None
|
|
|
|
|
|
@dataclass
|
|
class FlashInferMLASparseMetadata(SparseMLACommonMetadata):
|
|
"""Attention metadata for FlashInfer MLA Sparse backend."""
|
|
|
|
|
|
class FlashInferMLASparseMetadataBuilder(
|
|
SparseMLACommonMetadataBuilder[FlashInferMLASparseMetadata]
|
|
):
|
|
"""Builder for FlashInfer MLA Sparse attention metadata."""
|
|
|
|
metadata_cls = FlashInferMLASparseMetadata
|
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
|
|
|
def __init__(
|
|
self,
|
|
kv_cache_spec: AttentionSpec,
|
|
layer_names: list[str],
|
|
vllm_config: VllmConfig,
|
|
device: torch.device,
|
|
) -> None:
|
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
|
|
|
num_q_heads = vllm_config.model_config.get_num_attention_heads(
|
|
vllm_config.parallel_config
|
|
)
|
|
threshold = {8: 128, 16: 128, 32: 128, 64: 256, 128: 1024}.get(
|
|
num_q_heads, 1024
|
|
)
|
|
self._init_reorder_batch_threshold(
|
|
threshold,
|
|
supports_spec_as_decode=True,
|
|
supports_dcp_with_varlen=True,
|
|
)
|
|
|
|
|
|
class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilder):
|
|
"""Metadata builder for the SM100 TRT-LLM sparse MLA kernel."""
|
|
|
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
|
|
|
@classmethod
|
|
def get_varlen_cudagraph_max_query_len(
|
|
cls,
|
|
vllm_config: VllmConfig,
|
|
kv_cache_spec: KVCacheSpec,
|
|
) -> int | None:
|
|
# Decode uses device request boundaries; prefill metadata is not graph-safe.
|
|
return max_decode_query_len(vllm_config)
|
|
|
|
def __init__(
|
|
self,
|
|
kv_cache_spec: AttentionSpec,
|
|
layer_names: list[str],
|
|
vllm_config: VllmConfig,
|
|
device: torch.device,
|
|
) -> None:
|
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
|
self.supports_draft_decode_metadata_update = (
|
|
self.dcp_world_size == 1
|
|
and self.pcp_world_size == 1
|
|
and vllm_config.attention_config.hisparse_config is None
|
|
)
|
|
|
|
# Under DCP the workspace must hold the trtllm-gen softmax-stats slab.
|
|
# The buffer address is baked into CUDA graphs, so it has to reach its
|
|
# final size here, before the first forward/capture (no lazy regrow).
|
|
if self.dcp_world_size > 1:
|
|
num_q_heads = vllm_config.model_config.get_num_attention_heads(
|
|
vllm_config.parallel_config
|
|
)
|
|
_get_workspace_buffer(
|
|
device,
|
|
_required_workspace_bytes(
|
|
self.dcp_world_size,
|
|
num_q_heads,
|
|
vllm_config.scheduler_config.max_num_batched_tokens,
|
|
),
|
|
)
|
|
|
|
def update_draft_decode_metadata(
|
|
self, metadata: FlashInferMLASparseMetadata
|
|
) -> None:
|
|
num_tokens = metadata.num_decode_tokens
|
|
if num_tokens:
|
|
compute_token_to_req_indices(
|
|
metadata.query_start_loc,
|
|
metadata.req_id_per_token,
|
|
num_tokens,
|
|
num_tokens,
|
|
)
|
|
|
|
|
|
# Global workspace buffer (lazily initialized)
|
|
_fi_sparse_workspace: torch.Tensor | None = None
|
|
|
|
# trtllm-gen carves a softmax-stats slab from the workspace whenever LSE is
|
|
# requested (FlashInfer csrc/trtllm_fmha_kernel_launcher.cu, unchanged from
|
|
# v0.6.14 through current main):
|
|
# sizeof(float2) * num_qo_heads * batch_size * round_up(max_q_len, 256)
|
|
# + 1 MiB guard
|
|
# forward_mqa always passes q_len == 1, so each (head, token) pair costs
|
|
# round_up(1, 256) == 256 slots.
|
|
_TRTLLM_GEN_SOFTMAX_STAT_BYTES = 8 # sizeof(float2)
|
|
_TRTLLM_GEN_SOFTMAX_SLOTS_PER_TOKEN = 256 # round_up(q_len=1, 256)
|
|
_TRTLLM_GEN_SOFTMAX_GUARD_BYTES = 1024 * 1024
|
|
|
|
# Keep in sync with the VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE default in
|
|
# vllm/envs.py.
|
|
_DEFAULT_WORKSPACE_BUFFER_SIZE = 394 * 1024 * 1024
|
|
|
|
|
|
def compute_trtllm_sparse_mla_workspace_bytes(
|
|
base_workspace_bytes: int,
|
|
dcp_world_size: int,
|
|
num_heads_per_rank: int,
|
|
max_num_batched_tokens: int,
|
|
) -> int:
|
|
"""Workspace bytes needed by the trtllm-gen sparse MLA decode kernel.
|
|
|
|
Under DCP the query is all-gathered across the DCP group in the head dim
|
|
and the kernel is asked for LSE, so trtllm-gen carves the softmax-stats
|
|
slab described above from the workspace before its (batch-independent)
|
|
counter and scratch regions. ``base_workspace_bytes`` must stay available
|
|
for those regions, so the slab is added on top of it (see #50781).
|
|
|
|
Without DCP no LSE is requested, no slab is carved, and the base size is
|
|
returned unchanged.
|
|
"""
|
|
if dcp_world_size <= 1:
|
|
return base_workspace_bytes
|
|
softmax_bytes = (
|
|
_TRTLLM_GEN_SOFTMAX_STAT_BYTES
|
|
* (num_heads_per_rank * dcp_world_size)
|
|
* max_num_batched_tokens
|
|
* _TRTLLM_GEN_SOFTMAX_SLOTS_PER_TOKEN
|
|
+ _TRTLLM_GEN_SOFTMAX_GUARD_BYTES
|
|
)
|
|
return base_workspace_bytes + softmax_bytes
|
|
|
|
|
|
def _required_workspace_bytes(
|
|
dcp_world_size: int,
|
|
num_heads_per_rank: int,
|
|
max_num_batched_tokens: int,
|
|
) -> int:
|
|
"""Resolve the workspace size, honoring an explicit env override."""
|
|
computed = compute_trtllm_sparse_mla_workspace_bytes(
|
|
_DEFAULT_WORKSPACE_BUFFER_SIZE,
|
|
dcp_world_size,
|
|
num_heads_per_rank,
|
|
max_num_batched_tokens,
|
|
)
|
|
if not envs.is_set("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE"):
|
|
return computed
|
|
env_bytes = envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE
|
|
if env_bytes < computed:
|
|
logger.warning_once(
|
|
"VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE=%d is below the %d bytes "
|
|
"computed for the sparse MLA workspace with dcp_world_size=%d, "
|
|
"%d heads/rank and max_num_batched_tokens=%d. Respecting the "
|
|
"override, but the trtllm-gen kernel may crash with a workspace "
|
|
"overflow; set VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE=%d or unset "
|
|
"it to use the computed size.",
|
|
env_bytes,
|
|
computed,
|
|
dcp_world_size,
|
|
num_heads_per_rank,
|
|
max_num_batched_tokens,
|
|
computed,
|
|
)
|
|
return env_bytes
|
|
|
|
|
|
def _get_workspace_buffer(
|
|
device: torch.device, min_bytes: int | None = None
|
|
) -> torch.Tensor:
|
|
global _fi_sparse_workspace
|
|
required = (
|
|
min_bytes
|
|
if min_bytes is not None
|
|
else envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE
|
|
)
|
|
if _fi_sparse_workspace is None or _fi_sparse_workspace.numel() < required:
|
|
# FlashInfer's CuteDSL MLA-decode tactic requires an int8 workspace;
|
|
# the trtllm-gen path views it as uint8, so int8 is safe for all backends.
|
|
_fi_sparse_workspace = torch.zeros(
|
|
required,
|
|
dtype=torch.int8,
|
|
device=device,
|
|
)
|
|
return _fi_sparse_workspace
|
|
|
|
|
|
class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
|
|
"""FlashInfer MLA Sparse implementation.
|
|
|
|
Uses the TRT-LLM MLA kernel with sparse_mla_top_k parameter for
|
|
sparse attention computation.
|
|
"""
|
|
|
|
can_return_lse_for_decode: bool = True
|
|
supports_dcp: bool = True
|
|
lse_base_on_e: bool = False
|
|
|
|
def __init__(
|
|
self,
|
|
num_heads: int,
|
|
head_size: int,
|
|
scale: float,
|
|
num_kv_heads: int,
|
|
alibi_slopes: list[float] | None,
|
|
sliding_window: int | None,
|
|
kv_cache_dtype: str,
|
|
logits_soft_cap: float | None,
|
|
attn_type: str,
|
|
kv_sharing_target_layer_name: str | None,
|
|
# MLA Specific Arguments
|
|
topk_indices_buffer: torch.Tensor | None = None,
|
|
indexer: "Indexer | None" = None,
|
|
**mla_args,
|
|
) -> None:
|
|
unsupported_features = [alibi_slopes, sliding_window, logits_soft_cap]
|
|
if any(unsupported_features):
|
|
raise NotImplementedError(
|
|
"FlashInferMLASparseImpl does not support one of the following: "
|
|
"alibi_slopes, sliding_window, logits_soft_cap"
|
|
)
|
|
|
|
if attn_type != AttentionType.DECODER:
|
|
raise NotImplementedError(
|
|
"Encoder self-attention and "
|
|
"encoder/decoder cross-attention "
|
|
"are not implemented for "
|
|
"FlashInferMLASparseImpl"
|
|
)
|
|
|
|
super().__init__(
|
|
num_heads,
|
|
head_size,
|
|
scale,
|
|
num_kv_heads,
|
|
alibi_slopes,
|
|
sliding_window,
|
|
kv_cache_dtype,
|
|
logits_soft_cap,
|
|
attn_type,
|
|
kv_sharing_target_layer_name,
|
|
indexer=indexer,
|
|
topk_indices_buffer=topk_indices_buffer,
|
|
**mla_args,
|
|
)
|
|
|
|
self._workspace_buffer: torch.Tensor | None = None
|
|
self.bmm1_scale: float | None = None
|
|
self.bmm2_scale: float | None = None
|
|
|
|
# Native no-rope MLA additionally requires a per-query-token active
|
|
# top-k length tensor.
|
|
self.is_nope_mla = self.qk_rope_head_dim == 0
|
|
|
|
# fp8 query quantization is required when using fp8 kv_cache,
|
|
# as the TRTLLM-GEN sparse MLA kernel requires matching dtypes
|
|
# for query and kv_cache (mixed bf16+fp8 is not supported).
|
|
self.supports_quant_query_input = True
|
|
|
|
def _prepare_mqa_kernel(
|
|
self,
|
|
layer: AttentionLayer,
|
|
device: torch.device,
|
|
) -> None:
|
|
if self._workspace_buffer is None:
|
|
self._workspace_buffer = _get_workspace_buffer(device)
|
|
|
|
if self.bmm1_scale is None:
|
|
self.bmm1_scale = self.scale
|
|
if is_quantized_kv_cache(self.kv_cache_dtype):
|
|
self.bmm1_scale *= layer._q_scale_float * layer._k_scale_float
|
|
if self.bmm2_scale is None:
|
|
self.bmm2_scale = 1.0
|
|
if is_quantized_kv_cache(self.kv_cache_dtype):
|
|
self.bmm2_scale *= layer._k_scale_float
|
|
|
|
def autotune_hisparse_decode(self, layer: AttentionLayer) -> None:
|
|
"""Autotune the largest legal HiSparse decode batch."""
|
|
assert isinstance(self.index_group, HiSparseMLAIndexGroup)
|
|
cache = self.index_group.cache(self.index_group_index)
|
|
assert self.topk_indices_buffer is not None
|
|
|
|
runtime = cache.runtime
|
|
kv_cache = runtime.hot.attention_cache
|
|
num_tokens = runtime.max_num_reqs
|
|
topk_tokens = self.topk_indices_buffer.shape[1]
|
|
|
|
q_dtype = (
|
|
current_platform.fp8_dtype()
|
|
if is_quantized_kv_cache(self.kv_cache_dtype)
|
|
else kv_cache.dtype
|
|
)
|
|
q = torch.zeros(
|
|
(
|
|
num_tokens,
|
|
self.num_heads,
|
|
self.kv_lora_rank + self.qk_rope_head_dim,
|
|
),
|
|
dtype=q_dtype,
|
|
device=kv_cache.device,
|
|
)
|
|
topk_indices = (
|
|
torch.arange(
|
|
topk_tokens,
|
|
dtype=torch.int32,
|
|
device=kv_cache.device,
|
|
)
|
|
.expand(num_tokens, -1)
|
|
.contiguous()
|
|
)
|
|
seq_lens = torch.full(
|
|
(num_tokens,),
|
|
topk_tokens,
|
|
dtype=torch.int32,
|
|
device=kv_cache.device,
|
|
)
|
|
self._forward_mqa_kernel(
|
|
q,
|
|
kv_cache,
|
|
topk_indices,
|
|
seq_lens,
|
|
layer=layer,
|
|
block_size=runtime.hot.block_size,
|
|
is_decode=True,
|
|
)
|
|
|
|
def _forward_mqa_kernel(
|
|
self,
|
|
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
|
kv_cache: torch.Tensor,
|
|
topk_indices: torch.Tensor,
|
|
seq_lens: torch.Tensor,
|
|
*,
|
|
layer: AttentionLayer,
|
|
block_size: int,
|
|
is_decode: bool,
|
|
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
|
if isinstance(q, tuple):
|
|
ql_nope, q_pe = q
|
|
if q_pe.shape[-1] == 0 and ql_nope.is_contiguous():
|
|
q = ql_nope
|
|
else:
|
|
q = torch.cat(q, dim=-1)
|
|
|
|
self._prepare_mqa_kernel(layer, q.device)
|
|
assert self._workspace_buffer is not None
|
|
assert self.bmm1_scale is not None
|
|
assert self.bmm2_scale is not None
|
|
|
|
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
|
|
|
kv_rows, block_stride_rows = flat_kv_row_view(
|
|
kv_cache.view(q.dtype), block_size
|
|
)
|
|
# Block sizes and strides have been aligned to 32 rows.
|
|
kv_cache = kv_rows.view(
|
|
-1, math.gcd(block_stride_rows, block_size, 64), kv_rows.shape[-1]
|
|
)
|
|
|
|
# Single-token sparse decode. trtllm-gen requires the q_len_per_request
|
|
# dim, but the sparse attention mask is fully per-token (each query token
|
|
# carries its own top-k index row), so unsqueeze is sufficient and
|
|
# correct. The MTP/multi-token q_len grouping is a perf-only layout and is
|
|
# deferred until MTP is validated end-to-end for this backend.
|
|
query = q.unsqueeze(1)
|
|
block_tables = topk_indices.unsqueeze(1)
|
|
seq_lens_arg = seq_lens
|
|
|
|
# page_table width = topk buffer width, which kpool widens past
|
|
# index_topk (topk_tokens) and rounds up to a multiple of 128. The
|
|
# kernel treats sparse_mla_top_k as the page-table *capacity* and bounds
|
|
# the active per-query length by ``seq_lens`` (the compacted valid
|
|
# count), so the -1 padding slots past seq_lens are never attended to.
|
|
# Use the actual buffer width instead of the fixed topk_tokens, which
|
|
# mismatches the page_table when index_kpool > 1.
|
|
sparse_topk_capacity = topk_indices.shape[1]
|
|
|
|
extra_kwargs: dict[str, torch.Tensor] = {}
|
|
needs_empty_query_guard = self.need_to_return_lse_for_decode or isinstance(
|
|
self.index_group, HiSparseMLAIndexGroup
|
|
)
|
|
if self.is_nope_mla:
|
|
# Resident TP queries have nonempty selections. Preserve empty-query
|
|
# handling for DCP's local selections and host-backed HiSparse.
|
|
topk_lens = seq_lens
|
|
if needs_empty_query_guard:
|
|
topk_lens = prepare_sparse_mla_safe_lengths(topk_indices, seq_lens)
|
|
extra_kwargs["sparse_mla_top_k_lens"] = topk_lens
|
|
|
|
kernel_out = trtllm_batch_decode_with_kv_cache_mla(
|
|
query=query,
|
|
kv_cache=kv_cache.unsqueeze(1),
|
|
workspace_buffer=self._workspace_buffer,
|
|
qk_nope_head_dim=self.qk_nope_head_dim,
|
|
kv_lora_rank=self.kv_lora_rank,
|
|
qk_rope_head_dim=self.qk_rope_head_dim,
|
|
block_tables=block_tables,
|
|
seq_lens=seq_lens_arg,
|
|
max_seq_len=sparse_topk_capacity,
|
|
bmm1_scale=self.bmm1_scale,
|
|
bmm2_scale=self.bmm2_scale,
|
|
sparse_mla_top_k=sparse_topk_capacity,
|
|
return_lse=self.need_to_return_lse_for_decode,
|
|
**extra_kwargs,
|
|
)
|
|
if self.need_to_return_lse_for_decode:
|
|
assert isinstance(kernel_out, tuple)
|
|
o, lse = kernel_out
|
|
else:
|
|
assert isinstance(kernel_out, torch.Tensor)
|
|
o = kernel_out
|
|
lse = None
|
|
|
|
out = o.view(-1, o.shape[-2], o.shape[-1])
|
|
if lse is not None:
|
|
lse = self._normalize_lse(lse, out.shape[0], out.shape[1])
|
|
if self.is_nope_mla and needs_empty_query_guard:
|
|
empty_queries = seq_lens == 0
|
|
if lse is not None:
|
|
# DCP combine already suppresses outputs with zero LSE weight.
|
|
lse.masked_fill_(empty_queries[:, None], float("-inf"))
|
|
else:
|
|
out.masked_fill_(empty_queries[:, None, None], 0.0)
|
|
elif lse is not None:
|
|
empty_rows = (topk_indices == -1).all(dim=-1)
|
|
out.masked_fill_(empty_rows.view(-1, 1, 1), 0.0)
|
|
lse.masked_fill_(empty_rows.view(-1, 1), float("-inf"))
|
|
return out, lse
|
|
|
|
@staticmethod
|
|
def _normalize_lse(
|
|
lse: torch.Tensor,
|
|
num_tokens: int,
|
|
num_heads: int,
|
|
) -> torch.Tensor:
|
|
# FlashInfer returns the decode LSE either as 2D (num_tokens, num_heads)
|
|
# or 3D ((num_tokens, num_heads, 1) / (num_tokens, 1, num_heads)).
|
|
# Collapse all of these to the (num_tokens, num_heads) the shared DCP
|
|
# reducer expects.
|
|
if lse.dim() == 3:
|
|
if lse.shape[-1] == 1:
|
|
lse = lse.squeeze(-1)
|
|
elif lse.shape[1] == 1:
|
|
lse = lse.squeeze(1)
|
|
elif lse.shape[0] * lse.shape[1] == num_tokens:
|
|
lse = lse.reshape(num_tokens, lse.shape[-1])
|
|
if lse.shape != (num_tokens, num_heads):
|
|
raise RuntimeError(
|
|
"Unexpected FlashInfer sparse MLA LSE shape: "
|
|
f"{tuple(lse.shape)}, expected ({num_tokens}, {num_heads})."
|
|
)
|
|
return lse
|