Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
@@ -62,23 +62,22 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _resolve_dsv4_kv_cache_dtype(
|
||||
use_flashmla_fp8_layout: bool,
|
||||
use_fp8_ds_mla_layout: bool,
|
||||
kv_cache_dtype: str,
|
||||
cache_config: CacheConfig | None,
|
||||
) -> tuple[str, torch.dtype]:
|
||||
"""Map ``(layout, --kv-cache-dtype)`` to ``(cache_dtype_str, torch_dtype)``.
|
||||
|
||||
Both layouts are paged; they differ in the per-token block format. The
|
||||
FlashMLA fp8 layout (FlashMLA / ROCm Aiter) is the ``fp8_ds_mla`` format:
|
||||
UE8M0 block-scaled fp8 packed as ``uint8`` (the canonical ``fp8_ds_mla``
|
||||
string is written back onto ``cache_config`` so the page-size specs pick
|
||||
the 576B per-token slot). Otherwise (FlashInfer) each token's KV row is
|
||||
stored in its plain element dtype — bf16 or per-tensor FP8 E4M3.
|
||||
``fp8_ds_mla`` format is UE8M0 block-scaled fp8 packed as ``uint8`` (the
|
||||
canonical ``fp8_ds_mla`` string is written back onto ``cache_config`` so the
|
||||
page-size specs pick the 576B per-token slot). Plain-row backends store each
|
||||
token's KV row in its element dtype: bf16 or per-tensor FP8 E4M3.
|
||||
"""
|
||||
if use_flashmla_fp8_layout:
|
||||
if use_fp8_ds_mla_layout:
|
||||
# fp8_ds_mla block format: UE8M0 block-scaled fp8 packed as uint8.
|
||||
assert kv_cache_dtype.startswith("fp8"), (
|
||||
f"DeepseekV4 FlashMLA fp8 layout only supports fp8 kv-cache, "
|
||||
f"DeepseekV4 fp8_ds_mla layout only supports fp8 kv-cache, "
|
||||
f"got {kv_cache_dtype}"
|
||||
)
|
||||
if kv_cache_dtype != "fp8_ds_mla":
|
||||
@@ -100,18 +99,20 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
|
||||
The platform-specific sparse-MLA forward (``forward_mqa`` /
|
||||
``get_padded_num_q_heads`` / ``_o_proj`` / ``backend_cls``) is provided by a
|
||||
subclass — ``DeepseekV4FlashMLAAttention`` / ``DeepseekV4FlashInferMLAAttention``
|
||||
(CUDA) or ``DeepseekV4ROCMAiterMLAAttention`` (ROCm) — selected by the
|
||||
platform-specific deepseek_v4 model module. The base is never instantiated
|
||||
directly.
|
||||
subclass — ``DeepseekV4FlashMLAAttention`` /
|
||||
``DeepseekV4FlashInferSM120Attention`` /
|
||||
``DeepseekV4FlashInferMLAAttention`` (CUDA) or
|
||||
``DeepseekV4ROCMAiterMLAAttention`` (ROCm) — selected by the platform-specific
|
||||
deepseek_v4 model module. The base is never instantiated directly.
|
||||
"""
|
||||
|
||||
# Provided by the platform subclass.
|
||||
backend_cls: ClassVar[type[AttentionBackend]]
|
||||
# KV-cache per-token block format (both layouts are paged). True (default)
|
||||
# = FlashMLA / ROCm fp8_ds_mla (UE8M0 block-scaled fp8 packed as uint8);
|
||||
# False = FlashInfer plain bf16 / per-tensor fp8 KV row.
|
||||
use_flashmla_fp8_layout: ClassVar[bool] = True
|
||||
# = fp8_ds_mla (UE8M0 block-scaled fp8 packed as uint8); False = plain
|
||||
# bf16 / per-tensor fp8 KV row. Backends can override the instance hook when
|
||||
# a single attention class dispatches across arch-specific layouts.
|
||||
use_fp8_ds_mla_layout: ClassVar[bool] = True
|
||||
# Prefill is processed in fixed-size chunks; this bounds the bf16 kv-gather
|
||||
# workspace allocated in _forward_prefill and is also read by the dummy-run
|
||||
# path to pre-reserve that workspace.
|
||||
@@ -145,6 +146,10 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
"""Inverse-RoPE + wo_a + wo_b output projection (platform-specific)."""
|
||||
raise NotImplementedError
|
||||
|
||||
def _uses_fp8_ds_mla_layout(self) -> bool:
|
||||
"""Return whether this instance stores fp8 KV in fp8_ds_mla layout."""
|
||||
return self.use_fp8_ds_mla_layout
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -276,13 +281,10 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
)
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
|
||||
# Resolve the kv-cache dtype from this backend's block format (a
|
||||
# ClassVar set by the subclass): fp8_ds_mla (UE8M0 block-scaled fp8 as
|
||||
# uint8) for FlashMLA / ROCm, vs a plain bf16 / per-tensor fp8 row for
|
||||
# FlashInfer. The same resolution drives the SWA cache tensor dtype
|
||||
# below.
|
||||
# Resolve the kv-cache dtype from this backend's block format. The same
|
||||
# resolution drives the SWA cache tensor dtype below.
|
||||
self.kv_cache_dtype, self.kv_cache_torch_dtype = _resolve_dsv4_kv_cache_dtype(
|
||||
self.use_flashmla_fp8_layout, cache_config.cache_dtype, cache_config
|
||||
self._uses_fp8_ds_mla_layout(), cache_config.cache_dtype, cache_config
|
||||
)
|
||||
|
||||
self.swa_cache_layer = DeepseekV4SWACache(
|
||||
@@ -539,7 +541,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
|
||||
# kv is unchanged; attention reads kv solely via swa_kv_cache.
|
||||
if cache_dtype == torch.uint8:
|
||||
# Legacy FlashMLA UE8M0 paged path. Horizontally fused:
|
||||
# fp8_ds_mla UE8M0 paged path. Horizontally fused:
|
||||
# Q side: per-head RMSNorm (no weight) + GPT-J RoPE, zero-filling
|
||||
# the padding head slots; the kernel allocates and returns
|
||||
# the padded q tensor.
|
||||
@@ -557,10 +559,10 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
swa_metadata.block_size,
|
||||
)
|
||||
|
||||
# FlashInfer full-cache path: the [num_blocks, block_size, 512] cache
|
||||
# stores the KV row in its plain dtype (no Q padding). bf16 rewrites q
|
||||
# in place; per-tensor fp8 writes a separately-allocated fp8 q and
|
||||
# quantizes the KV row.
|
||||
# Plain-row path: the [num_blocks, block_size, 512] cache stores the KV
|
||||
# row in its element dtype (no Q padding). bf16 rewrites q in place;
|
||||
# per-tensor fp8 writes a separately-allocated fp8 q and quantizes the
|
||||
# KV row.
|
||||
block_size = swa_metadata.block_size
|
||||
swa_kv_cache_3d = swa_kv_cache.view(-1, block_size, self.head_dim)
|
||||
if cache_dtype == torch.bfloat16:
|
||||
@@ -601,18 +603,18 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
self.compress_ratio <= 1
|
||||
): # SWA part. Allocated separately as DeepseekV4SWACache.
|
||||
return None
|
||||
# FlashMLA uses the fp8_ds_mla block format (UE8M0 block-scaled fp8 as
|
||||
# uint8, 576B aligned); FlashInfer stores a plain bf16 / per-tensor fp8
|
||||
# row with no extra alignment.
|
||||
is_flashmla = self.kv_cache_dtype == "fp8_ds_mla"
|
||||
# fp8_ds_mla is a UE8M0 block-scaled uint8 layout and needs 576B
|
||||
# alignment; plain bf16 / per-tensor fp8 rows use natural element-size
|
||||
# pages.
|
||||
uses_fp8_ds_mla_layout = self.kv_cache_dtype == "fp8_ds_mla"
|
||||
return MLAAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=torch.uint8 if is_flashmla else self.kv_cache_torch_dtype,
|
||||
dtype=torch.uint8 if uses_fp8_ds_mla_layout else self.kv_cache_torch_dtype,
|
||||
compress_ratio=self.compress_ratio,
|
||||
cache_dtype_str=self.kv_cache_dtype,
|
||||
alignment=576 if is_flashmla else None, # FlashMLA needs 576B
|
||||
alignment=576 if uses_fp8_ds_mla_layout else None,
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
|
||||
|
||||
@@ -40,9 +40,18 @@ def _fused_inv_rope_fp8_quant_per_head(
|
||||
USE_GDC: tl.constexpr,
|
||||
launch_pdl: tl.constexpr, # triton metadata
|
||||
):
|
||||
# int64: stride multiply overflows int32 past num_tokens=32768 (IMA).
|
||||
# Cast every stride to int64 — without this, Python-int strides are
|
||||
# inferred as int32 and `pid_token(int64) × stride(int32)` can lower to
|
||||
# int32 arithmetic, wrapping past 2³¹ for large prefill batches → IMA.
|
||||
pid_token = tl.program_id(0).to(tl.int64)
|
||||
pid_gh = tl.program_id(1).to(tl.int64)
|
||||
o_stride_token = o_stride_token.to(tl.int64)
|
||||
o_stride_head = o_stride_head.to(tl.int64)
|
||||
cache_stride_pos = cache_stride_pos.to(tl.int64)
|
||||
fp8_stride_group = fp8_stride_group.to(tl.int64)
|
||||
fp8_stride_token = fp8_stride_token.to(tl.int64)
|
||||
scale_stride_group = scale_stride_group.to(tl.int64)
|
||||
scale_stride_k = scale_stride_k.to(tl.int64)
|
||||
|
||||
g = pid_gh // heads_per_group
|
||||
head_in_group = pid_gh % heads_per_group
|
||||
|
||||
@@ -155,17 +155,17 @@ class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
|
||||
raise ValueError(f"Invalid compress ratio: {compress_ratio}")
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# FlashMLA's UE8M0 paged layout needs 576B alignment; the FlashInfer
|
||||
# full-cache path shares state pages with contiguous KV pages, so
|
||||
# padding would break page matching.
|
||||
is_flashmla = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
# fp8_ds_mla is the UE8M0 paged layout and needs 576B alignment. Plain
|
||||
# full-cache rows share state pages with contiguous KV pages, so padding
|
||||
# would break page matching.
|
||||
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
return SlidingWindowMLASpec( # only has one vector instead of K + V
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.state_dim,
|
||||
dtype=self.dtype,
|
||||
sliding_window=self.sliding_window,
|
||||
alignment=576 if is_flashmla else None,
|
||||
alignment=576 if uses_fp8_ds_mla_layout else None,
|
||||
)
|
||||
|
||||
def forward(self): ...
|
||||
@@ -340,8 +340,8 @@ class DeepseekCompressor(nn.Module):
|
||||
k_cache_layer = self._static_forward_context[self.k_cache_prefix]
|
||||
kv_cache = k_cache_layer.kv_cache
|
||||
|
||||
# FlashInfer V4 reads a contiguous bf16 / per-tensor fp8 cache row; the
|
||||
# legacy FlashMLA path uses the UE8M0 paged uint8 layout.
|
||||
# Plain-row V4 reads a contiguous bf16 / per-tensor fp8 cache row; the
|
||||
# fp8_ds_mla path uses the UE8M0 paged uint8 layout.
|
||||
store_full_kv = self.head_dim == 512 and kv_cache.dtype != torch.uint8
|
||||
store_full_fp8 = kv_cache.dtype == torch.float8_e4m3fn
|
||||
fp8_scale = (
|
||||
@@ -358,8 +358,8 @@ class DeepseekCompressor(nn.Module):
|
||||
compress_norm_rope_store_cutedsl,
|
||||
)
|
||||
|
||||
# head=512 on CUDA always uses cutedsl, for both the legacy UE8M0
|
||||
# layout and the FlashInfer full-cache layout. The full-cache flags
|
||||
# head=512 on CUDA always uses cutedsl, for both the fp8_ds_mla
|
||||
# layout and the plain full-cache layout. The full-cache flags
|
||||
# are consumed only here.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_cutedsl
|
||||
extra_kwargs: dict[str, Any] = dict(
|
||||
|
||||
@@ -1,13 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V4 FlashInfer TRTLLM-gen sparse MLA backend.
|
||||
|
||||
Uses FlashInfer's public ``trtllm_batch_decode_sparse_mla_dsv4`` launcher with a
|
||||
plain bf16 / per-tensor FP8 KV row (vs FlashMLA's packed ``fp8_ds_mla`` block
|
||||
format). Shares the V4 sparse-index pipeline (SWA cache + compressor + indexer,
|
||||
256-token blocks, head_size 512) with the FlashMLA V4 backend; only the
|
||||
attention forward differs.
|
||||
"""
|
||||
"""DeepSeek V4 FlashInfer sparse MLA backend."""
|
||||
|
||||
from typing import TYPE_CHECKING, ClassVar, cast
|
||||
|
||||
@@ -18,6 +11,7 @@ from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.common.ops import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
compute_global_topk_indices_and_lens,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
|
||||
compute_fp8_einsum_recipe,
|
||||
@@ -27,13 +21,14 @@ from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4FlashMLABackend,
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.flashinfer import flashinfer_trtllm_batch_decode_sparse_mla_dsv4
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
# 128 MB TRTLLM-gen workspace, allocated once per device and zero-initialized
|
||||
# (required for first use). Reused across all FlashInfer V4 layers.
|
||||
_FLASHINFER_DSV4_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
|
||||
_flashinfer_dsv4_workspace_by_device: dict[torch.device, torch.Tensor] = {}
|
||||
|
||||
@@ -51,34 +46,113 @@ def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
|
||||
|
||||
|
||||
class DeepseekV4FlashInferMLASparseBackend(DeepseekV4FlashMLABackend):
|
||||
"""Shares the FlashMLA V4 metadata/cache pipeline; swaps the attention impl.
|
||||
"""FlashInfer backend using the DSv4 sparse metadata/cache layout.
|
||||
|
||||
Inheriting from the FlashMLA V4 backend reuses its ``DeepseekV4FlashMLAMetadata``
|
||||
builder.
|
||||
Inheriting from the FlashMLA V4 backend reuses its
|
||||
``DeepseekV4FlashMLAMetadata`` builder.
|
||||
"""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16", "fp8"]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"bfloat16",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
"fp8_ds_mla",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
return [256]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHINFER_MLA_SPARSE_DSV4"
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return [512]
|
||||
|
||||
@classmethod
|
||||
def supports_sink(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def is_sparse(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
||||
return capability.major in [10, 12]
|
||||
|
||||
@classmethod
|
||||
def supports_combination(
|
||||
cls,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType | None,
|
||||
block_size: int | None,
|
||||
use_mla: bool,
|
||||
has_sink: bool,
|
||||
use_sparse: bool,
|
||||
use_mm_prefix: bool,
|
||||
device_capability: DeviceCapability,
|
||||
) -> str | None:
|
||||
if device_capability.major == 10:
|
||||
if kv_cache_dtype == "fp8_ds_mla":
|
||||
return (
|
||||
"FLASHINFER_MLA_SPARSE_DSV4 SM10x uses the plain "
|
||||
"per-tensor FP8 KV layout, not fp8_ds_mla"
|
||||
)
|
||||
if kv_cache_dtype not in (None, "auto", "bfloat16", "fp8", "fp8_e4m3"):
|
||||
return "kv_cache_dtype not supported"
|
||||
return None
|
||||
if device_capability.major == 12:
|
||||
if kv_cache_dtype not in ("fp8", "fp8_e4m3", "fp8_ds_mla"):
|
||||
return "kv_cache_dtype not supported"
|
||||
from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120
|
||||
|
||||
if not has_flashinfer_sparse_mla_sm120():
|
||||
return (
|
||||
"FLASHINFER_MLA_SPARSE_DSV4 SM120 requires FlashInfer's "
|
||||
"sparse MLA decode API"
|
||||
)
|
||||
return None
|
||||
return "FLASHINFER_MLA_SPARSE_DSV4 requires SM10x or SM12x"
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
device_capability = current_platform.get_device_capability()
|
||||
if device_capability is not None and device_capability.major == 12:
|
||||
return DeepseekV4FlashMLABackend.get_kv_cache_shape(
|
||||
num_blocks,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
cache_dtype_str,
|
||||
)
|
||||
assert num_kv_heads == 1
|
||||
return (num_blocks, block_size, head_size)
|
||||
|
||||
|
||||
class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
"""FlashInfer TRTLLM-gen sparse MLA attention layer for DeepSeek V4."""
|
||||
"""FlashInfer TRTLLM-gen sparse MLA attention layer for SM100 DeepSeek V4."""
|
||||
|
||||
backend_cls = DeepseekV4FlashInferMLASparseBackend
|
||||
# FlashInfer stores a plain bf16 / per-tensor fp8 KV row, not the FlashMLA
|
||||
# packed fp8_ds_mla block format (UE8M0 block-scaled fp8 as uint8).
|
||||
use_flashmla_fp8_layout: ClassVar[bool] = False
|
||||
use_fp8_ds_mla_layout: ClassVar[bool] = False
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
# FP8 decode kernel only supports h_q = 64 or 128.
|
||||
if num_heads > 128:
|
||||
raise ValueError(
|
||||
f"DeepseekV4 Flashinfer MLA Sparse does not support {num_heads} heads "
|
||||
f"DeepseekV4 FlashInfer MLA Sparse does not support {num_heads} heads "
|
||||
"(FP8 decode kernel requires h_q in {64, 128})."
|
||||
)
|
||||
return 64 if num_heads <= 64 else 128
|
||||
@@ -106,8 +180,6 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
# per-tensor FP8 cache path consumes these; bf16 reads ``self.scale``.
|
||||
if self.kv_cache_torch_dtype != torch.float8_e4m3fn:
|
||||
return
|
||||
# TODO: load real per-tensor Q/KV scales from the checkpoint; unit
|
||||
# scales until the scale tensor names are wired.
|
||||
fp8_q_scale = 1.0
|
||||
fp8_kv_scale = 1.0
|
||||
self.register_buffer(
|
||||
@@ -125,9 +197,8 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
torch.tensor([fp8_kv_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
# TRTLLM-gen takes scalar scale args on a distinct (correct) C++ path
|
||||
# vs 1-elem tensors, so these are Python floats. bmm1 folds the softmax
|
||||
# scale and the Q/KV per-tensor scales; bmm2 is the KV scale.
|
||||
# TRTLLM-gen takes scalar scale args on a distinct C++ path vs
|
||||
# one-element tensors, so these are Python floats.
|
||||
self._flashinfer_fp8_bmm1_scale = self.scale * fp8_q_scale * fp8_kv_scale
|
||||
self._flashinfer_fp8_bmm2_scale = fp8_kv_scale
|
||||
|
||||
@@ -387,9 +458,8 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
|
||||
assert query_start_loc is not None and query_start_loc_cpu is not None
|
||||
|
||||
# Keep Perkz's two-call decode/prefill split: the TRTLLM-gen launcher is
|
||||
# tuned for uniform-q batches, and collapsing the mixed batch into a
|
||||
# single call is the suspected source of the prior IMA.
|
||||
# Keep the TRTLLM-gen decode/prefill split: the launcher is tuned for
|
||||
# uniform-q batches, and this avoids flattening mixed batches into one call.
|
||||
if num_decode_tokens > 0:
|
||||
decode_cu = query_start_loc[: num_decodes + 1]
|
||||
decode_cu_cpu = query_start_loc_cpu[: num_decodes + 1]
|
||||
@@ -434,3 +504,379 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
cum_seq_lens_q=prefill_cu,
|
||||
max_q_len=int(prefill_lens_cpu.max().item()),
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
|
||||
"""DeepSeek V4 sparse MLA attention through FlashInfer's SM120 kernels."""
|
||||
|
||||
backend_cls = DeepseekV4FlashInferMLASparseBackend
|
||||
use_fp8_ds_mla_layout: ClassVar[bool] = True
|
||||
|
||||
@staticmethod
|
||||
def _get_workspace(device: torch.device) -> torch.Tensor:
|
||||
return _get_flashinfer_dsv4_workspace(device)
|
||||
|
||||
@staticmethod
|
||||
def _as_sparse_cache(kv_cache: torch.Tensor) -> torch.Tensor:
|
||||
if kv_cache.dtype == torch.float8_e4m3fn:
|
||||
kv_cache = kv_cache.view(torch.uint8)
|
||||
if kv_cache.dim() == 4:
|
||||
return kv_cache
|
||||
return kv_cache.unsqueeze(-2)
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
if num_heads <= 16:
|
||||
return 16
|
||||
if num_heads <= 32:
|
||||
return 32
|
||||
if num_heads <= 64:
|
||||
return 64
|
||||
if num_heads <= 128:
|
||||
return 128
|
||||
raise ValueError(
|
||||
f"DeepseekV4 FlashInfer MLA Sparse does not support {num_heads} heads "
|
||||
"(SM120 kernel requires h_q in {16, 32, 64, 128})."
|
||||
)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.wo_a,
|
||||
self.wo_b,
|
||||
n_groups=self.n_local_groups,
|
||||
heads_per_group=self.n_local_heads // self.n_local_groups,
|
||||
nope_dim=self.nope_head_dim,
|
||||
rope_dim=self.rope_head_dim,
|
||||
o_lora_rank=self.o_lora_rank,
|
||||
einsum_recipe=self._einsum_recipe,
|
||||
tma_aligned_scales=self._tma_aligned_scales,
|
||||
)
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120
|
||||
|
||||
if not has_flashinfer_sparse_mla_sm120():
|
||||
raise RuntimeError(
|
||||
"FLASHINFER_MLA_SPARSE_DSV4 on SM120 requires FlashInfer's "
|
||||
"sparse MLA decode API."
|
||||
)
|
||||
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe()
|
||||
# Per-tensor FP8 cache path scales.
|
||||
if self.kv_cache_torch_dtype != torch.float8_e4m3fn:
|
||||
return
|
||||
fp8_q_scale = 1.0
|
||||
fp8_kv_scale = 1.0
|
||||
self.register_buffer(
|
||||
"_flashinfer_fp8_q_scale",
|
||||
torch.tensor([fp8_q_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
self.register_buffer(
|
||||
"_flashinfer_fp8_q_scale_inv",
|
||||
torch.tensor([1.0 / fp8_q_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
self.register_buffer(
|
||||
"_flashinfer_fp8_kv_scale",
|
||||
torch.tensor([fp8_kv_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
# FlashInfer expects scalar scale arguments for this path.
|
||||
self._flashinfer_fp8_bmm1_scale = self.scale * fp8_q_scale * fp8_kv_scale
|
||||
self._flashinfer_fp8_bmm2_scale = fp8_kv_scale
|
||||
|
||||
def _reserve_empty_forward_workspace(self) -> None:
|
||||
self._get_workspace(
|
||||
torch.device("cuda", torch.accelerator.current_device_index())
|
||||
)
|
||||
|
||||
def _forward_sparse_impl(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
self_kv_cache: torch.Tensor | None,
|
||||
swa_kv_cache: torch.Tensor,
|
||||
swa_only: bool,
|
||||
) -> None:
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
if swa_metadata.num_prefills > 0:
|
||||
self._forward_prefill(
|
||||
q=q[num_decode_tokens:],
|
||||
compressed_k_cache=self_kv_cache,
|
||||
swa_k_cache=swa_kv_cache,
|
||||
output=output[num_decode_tokens:],
|
||||
attn_metadata=flashmla_metadata,
|
||||
swa_metadata=swa_metadata,
|
||||
)
|
||||
if swa_metadata.num_decodes > 0:
|
||||
self._forward_decode(
|
||||
q=q[:num_decode_tokens],
|
||||
kv_cache=self_kv_cache,
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=flashmla_metadata,
|
||||
swa_only=swa_only,
|
||||
output=output[:num_decode_tokens],
|
||||
)
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
# Output may be padded to backend-supported head counts.
|
||||
assert output.shape[0] == q.shape[0] and output.shape[-1] == q.shape[-1], (
|
||||
f"output buffer shape {output.shape} incompatible with q shape {q.shape}"
|
||||
)
|
||||
assert output.shape[1] >= q.shape[1], (
|
||||
f"output heads {output.shape[1]} must be >= q heads {q.shape[1]}"
|
||||
)
|
||||
# Per-tensor FP8 q produces a bf16 attention output.
|
||||
expected_output_dtype = (
|
||||
torch.bfloat16 if q.dtype == torch.float8_e4m3fn else q.dtype
|
||||
)
|
||||
assert output.dtype == expected_output_dtype, (
|
||||
f"output dtype {output.dtype} must match expected {expected_output_dtype} "
|
||||
f"for q dtype {q.dtype}"
|
||||
)
|
||||
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if attn_metadata is None:
|
||||
self._reserve_empty_forward_workspace()
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
flashmla_metadata = cast(
|
||||
DeepseekV4FlashMLAMetadata | None, attn_metadata.get(self.prefix)
|
||||
)
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata | None",
|
||||
attn_metadata.get(self.swa_cache_layer.prefix),
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
|
||||
swa_only = self.compress_ratio <= 1
|
||||
# SWA-only layers don't allocate their own compressed KV cache.
|
||||
self_kv_cache = self.kv_cache if not swa_only else None
|
||||
swa_kv_cache = self.swa_cache_layer.kv_cache
|
||||
|
||||
self._forward_sparse_impl(
|
||||
q=q,
|
||||
output=output,
|
||||
flashmla_metadata=flashmla_metadata,
|
||||
swa_metadata=swa_metadata,
|
||||
self_kv_cache=self_kv_cache,
|
||||
swa_kv_cache=swa_kv_cache,
|
||||
swa_only=swa_only,
|
||||
)
|
||||
|
||||
def _prepare_query(self, q: torch.Tensor, output: torch.Tensor) -> torch.Tensor:
|
||||
if self.kv_cache_torch_dtype == torch.float8_e4m3fn:
|
||||
assert q.dtype == torch.float8_e4m3fn
|
||||
q = q.to(torch.bfloat16)
|
||||
else:
|
||||
assert q.dtype == torch.bfloat16
|
||||
padded_heads = output.shape[1]
|
||||
if q.shape[1] < padded_heads:
|
||||
padded_query = q.new_zeros((q.shape[0], padded_heads, q.shape[2]))
|
||||
padded_query[:, : q.shape[1], :] = q
|
||||
q = padded_query
|
||||
return q.contiguous()
|
||||
|
||||
def _forward_decode(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_only: bool,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
extra_sparse_indices = None
|
||||
extra_sparse_lengths = None
|
||||
if not swa_only:
|
||||
if attn_metadata is None:
|
||||
raise RuntimeError(
|
||||
"Sparse MLA metadata is required for compressed layers."
|
||||
)
|
||||
if swa_metadata.is_valid_token is None:
|
||||
raise RuntimeError(
|
||||
"SWA validity metadata is required for compressed layers."
|
||||
)
|
||||
is_valid = swa_metadata.is_valid_token[:num_decode_tokens]
|
||||
if self.compress_ratio == 4:
|
||||
if self.topk_indices_buffer is None:
|
||||
raise RuntimeError(
|
||||
"C4A decode requires top-k indices from the indexer."
|
||||
)
|
||||
block_size = attn_metadata.block_size // self.compress_ratio
|
||||
global_indices, extra_sparse_lengths = (
|
||||
compute_global_topk_indices_and_lens(
|
||||
self.topk_indices_buffer[:num_decode_tokens],
|
||||
swa_metadata.token_to_req_indices,
|
||||
attn_metadata.block_table[:num_decodes],
|
||||
block_size,
|
||||
is_valid,
|
||||
)
|
||||
)
|
||||
extra_sparse_indices = global_indices.view(num_decode_tokens, 1, -1)
|
||||
else:
|
||||
extra_sparse_indices = attn_metadata.c128a_global_decode_topk_indices
|
||||
extra_sparse_lengths = attn_metadata.c128a_decode_topk_lens
|
||||
|
||||
swa_indices = swa_metadata.decode_swa_indices
|
||||
swa_lens = swa_metadata.decode_swa_lens
|
||||
assert swa_indices is not None
|
||||
assert swa_lens is not None
|
||||
q = self._prepare_query(q, output)
|
||||
swa_cache = self._as_sparse_cache(self.swa_cache_layer.kv_cache)
|
||||
extra_cache = self._as_sparse_cache(kv_cache) if kv_cache is not None else None
|
||||
if extra_cache is not None and extra_sparse_indices is None:
|
||||
raise RuntimeError(
|
||||
"Compressed sparse MLA decode requires compressed sparse indices."
|
||||
)
|
||||
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
|
||||
query=q,
|
||||
swa_kv_cache=swa_cache,
|
||||
workspace_buffer=self._get_workspace(q.device),
|
||||
sparse_indices=swa_indices,
|
||||
compressed_kv_cache=extra_cache,
|
||||
out=output,
|
||||
bmm1_scale=self.scale,
|
||||
sinks=self.attn_sink,
|
||||
kv_layout="NHD",
|
||||
swa_topk_lens=swa_lens,
|
||||
extra_sparse_indices=extra_sparse_indices,
|
||||
extra_sparse_topk_lens=extra_sparse_lengths,
|
||||
)
|
||||
|
||||
def _forward_prefill(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
compressed_k_cache: torch.Tensor | None,
|
||||
swa_k_cache: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
) -> None:
|
||||
swa_only = self.compress_ratio <= 1
|
||||
|
||||
num_prefills = swa_metadata.num_prefills
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
num_prefill_tokens = swa_metadata.num_prefill_tokens
|
||||
|
||||
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
|
||||
assert query_start_loc_cpu is not None
|
||||
prefill_token_base = query_start_loc_cpu[num_decodes]
|
||||
|
||||
local_topk_indices: torch.Tensor | None
|
||||
if swa_only:
|
||||
local_topk_indices = None
|
||||
elif self.compress_ratio == 4:
|
||||
if self.topk_indices_buffer is None:
|
||||
raise RuntimeError(
|
||||
"C4A prefill requires top-k indices from the indexer."
|
||||
)
|
||||
local_topk_indices = self.topk_indices_buffer[
|
||||
num_decode_tokens : num_decode_tokens + num_prefill_tokens
|
||||
]
|
||||
else:
|
||||
if attn_metadata is None:
|
||||
raise RuntimeError("C128A prefill metadata is missing.")
|
||||
local_topk_indices = attn_metadata.c128a_prefill_topk_indices
|
||||
|
||||
extra_sparse_indices: torch.Tensor | None = None
|
||||
extra_sparse_lengths: torch.Tensor | None = None
|
||||
if local_topk_indices is not None:
|
||||
if attn_metadata is None:
|
||||
raise RuntimeError("C4A prefill metadata is missing.")
|
||||
if swa_metadata.token_to_req_indices is None:
|
||||
raise RuntimeError("C4A prefill request mapping is missing.")
|
||||
if swa_metadata.is_valid_token is None:
|
||||
raise RuntimeError("C4A prefill validity metadata is missing.")
|
||||
prefill_token_slice = slice(
|
||||
num_decode_tokens, num_decode_tokens + num_prefill_tokens
|
||||
)
|
||||
block_size = attn_metadata.block_size // self.compress_ratio
|
||||
extra_sparse_indices, extra_sparse_lengths = (
|
||||
compute_global_topk_indices_and_lens(
|
||||
local_topk_indices,
|
||||
swa_metadata.token_to_req_indices[prefill_token_slice],
|
||||
attn_metadata.block_table,
|
||||
block_size,
|
||||
swa_metadata.is_valid_token[prefill_token_slice],
|
||||
)
|
||||
)
|
||||
|
||||
assert swa_metadata.prefill_swa_indices is not None
|
||||
assert swa_metadata.prefill_swa_lens is not None
|
||||
|
||||
q = self._prepare_query(q, output)
|
||||
swa_kv_paged = self._as_sparse_cache(swa_k_cache)
|
||||
if swa_only:
|
||||
extra_kv_paged = None
|
||||
else:
|
||||
if compressed_k_cache is None:
|
||||
raise RuntimeError(
|
||||
"Compressed sparse MLA layers require their compressed KV cache."
|
||||
)
|
||||
extra_kv_paged = self._as_sparse_cache(compressed_k_cache)
|
||||
|
||||
num_chunks = (
|
||||
num_prefills + self.PREFILL_CHUNK_SIZE - 1
|
||||
) // self.PREFILL_CHUNK_SIZE
|
||||
for chunk_idx in range(num_chunks):
|
||||
chunk_start = chunk_idx * self.PREFILL_CHUNK_SIZE
|
||||
chunk_end = min(chunk_start + self.PREFILL_CHUNK_SIZE, num_prefills)
|
||||
query_start = (
|
||||
query_start_loc_cpu[num_decodes + chunk_start] - prefill_token_base
|
||||
)
|
||||
query_end = (
|
||||
query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base
|
||||
)
|
||||
|
||||
extra_sparse_indices_chunk = (
|
||||
extra_sparse_indices[query_start:query_end]
|
||||
if extra_sparse_indices is not None
|
||||
else None
|
||||
)
|
||||
extra_sparse_lengths_chunk = (
|
||||
extra_sparse_lengths[query_start:query_end]
|
||||
if extra_sparse_lengths is not None
|
||||
else None
|
||||
)
|
||||
|
||||
q_chunk = q[query_start:query_end]
|
||||
swa_indices_chunk = swa_metadata.prefill_swa_indices[query_start:query_end]
|
||||
swa_lens_chunk = swa_metadata.prefill_swa_lens[query_start:query_end]
|
||||
if extra_kv_paged is not None and extra_sparse_indices_chunk is None:
|
||||
raise RuntimeError(
|
||||
"Compressed sparse MLA prefill requires compressed sparse indices."
|
||||
)
|
||||
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
|
||||
query=q_chunk,
|
||||
swa_kv_cache=swa_kv_paged,
|
||||
workspace_buffer=self._get_workspace(q.device),
|
||||
sparse_indices=swa_indices_chunk,
|
||||
compressed_kv_cache=extra_kv_paged,
|
||||
out=output[query_start:query_end],
|
||||
bmm1_scale=self.scale,
|
||||
sinks=self.attn_sink,
|
||||
kv_layout="NHD",
|
||||
swa_topk_lens=swa_lens_chunk,
|
||||
extra_sparse_indices=extra_sparse_indices_chunk,
|
||||
extra_sparse_topk_lens=extra_sparse_lengths_chunk,
|
||||
)
|
||||
|
||||
@@ -60,9 +60,11 @@ from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
|
||||
DeepseekV4FlashInferMLAAttention,
|
||||
DeepseekV4FlashInferSM120Attention,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.flashmla import DeepseekV4FlashMLAAttention
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -736,14 +738,34 @@ class DeepseekV4MoE(nn.Module):
|
||||
def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
"""Pick the CUDA sparse-MLA attention class for the configured backend.
|
||||
|
||||
An explicit ``--attention-backend FLASHINFER_MLA_SPARSE_DSV4`` selects the
|
||||
FlashInfer TRTLLM-gen path; otherwise the FlashMLA path is used.
|
||||
The generic CUDA backend selector does not instantiate DSv4 layers directly,
|
||||
so map generic sparse-MLA choices to the DSv4-specialized attention class.
|
||||
Without an explicit backend, SM12 defaults to FlashInfer while the other
|
||||
CUDA arches keep the FlashMLA path.
|
||||
"""
|
||||
if (
|
||||
vllm_config.attention_config.backend
|
||||
== AttentionBackendEnum.FLASHINFER_MLA_SPARSE_DSV4
|
||||
backend = vllm_config.attention_config.backend
|
||||
device_capability = current_platform.get_device_capability()
|
||||
if backend in (
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM120,
|
||||
):
|
||||
raise ValueError(
|
||||
f"{backend.name} is not a DeepSeek V4 attention backend. "
|
||||
"Use FLASHINFER_MLA_SPARSE_DSV4 for DeepSeek V4 FlashInfer "
|
||||
"sparse MLA."
|
||||
)
|
||||
if backend == AttentionBackendEnum.FLASHINFER_MLA_SPARSE_DSV4:
|
||||
if device_capability is not None and device_capability.major == 12:
|
||||
return DeepseekV4FlashInferSM120Attention
|
||||
return DeepseekV4FlashInferMLAAttention
|
||||
if backend in (
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE_DSV4,
|
||||
):
|
||||
return DeepseekV4FlashMLAAttention
|
||||
|
||||
if device_capability is not None and device_capability.major == 12:
|
||||
return DeepseekV4FlashInferSM120Attention
|
||||
return DeepseekV4FlashMLAAttention
|
||||
|
||||
|
||||
|
||||
@@ -86,6 +86,10 @@ class DeepseekV4FlashMLABackend(AttentionBackend):
|
||||
def is_sparse(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_sink(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
||||
return capability.major in [9, 10]
|
||||
|
||||
Reference in New Issue
Block a user