From f4359a70f9e04b0223ef9209db6f0d4d6a10f094 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Mon, 15 Jun 2026 17:14:51 -0700 Subject: [PATCH] [DSV4][Minor] Fix supported KV cache dtypes (#44892) Signed-off-by: Woosuk Kwon --- docs/design/attention_backends.md | 4 ++-- vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py | 11 ++++++----- vllm/models/deepseek_v4/sparse_mla.py | 1 - 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/design/attention_backends.md b/docs/design/attention_backends.md index fcf05cf685..6f8feeb887 100644 --- a/docs/design/attention_backends.md +++ b/docs/design/attention_backends.md @@ -252,6 +252,6 @@ default on NVIDIA is `FLASHMLA_SPARSE_DSV4`. | Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | Sparse | MM Prefix | DCP | Attention Types | Compute Cap. | | ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ | -| `FLASHINFER_MLA_SPARSE_DSV4` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | Any | -| `FLASHMLA_SPARSE_DSV4` | bf16 | `auto`, `bfloat16`, `fp8_ds_mla`, `fp8` | 256 | 512 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x | +| `FLASHINFER_MLA_SPARSE_DSV4` | bf16 | `auto`, `bfloat16`, `fp8` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | Any | +| `FLASHMLA_SPARSE_DSV4` | bf16 | `auto`, `fp8_ds_mla`, `fp8` | 256 | 512 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x | | `ROCM_FLASHMLA_SPARSE_DSV4` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A | diff --git a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py index d036943d47..a357edf554 100644 --- a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, ClassVar, cast import torch +from vllm.config.cache import CacheDType from vllm.forward_context import get_forward_context from vllm.models.deepseek_v4.attention import DeepseekV4Attention from vllm.models.deepseek_v4.common.ops import ( @@ -52,13 +53,13 @@ def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor: class DeepseekV4FlashInferMLASparseBackend(DeepseekV4FlashMLABackend): """Shares the FlashMLA V4 metadata/cache pipeline; swaps the attention impl. - Inheriting from the FlashMLA V4 backend reuses its - ``DeepseekV4FlashMLAMetadata`` builder (which the V4 sparse-index - pipeline needs — the V3.2 FlashInfer builder lacks the ``c128a_*`` fields), - 256-token blocks, head_size 512, and the (num_blocks, block_size, 512) cache - shape for non-``fp8_ds_mla`` dtypes. + Inheriting from the FlashMLA V4 backend reuses its ``DeepseekV4FlashMLAMetadata`` + builder. """ + supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] + supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16", "fp8"] + @staticmethod def get_name() -> str: return "FLASHINFER_MLA_SPARSE_DSV4" diff --git a/vllm/models/deepseek_v4/sparse_mla.py b/vllm/models/deepseek_v4/sparse_mla.py index bf6d29f0a2..ca14fe20b1 100644 --- a/vllm/models/deepseek_v4/sparse_mla.py +++ b/vllm/models/deepseek_v4/sparse_mla.py @@ -46,7 +46,6 @@ class DeepseekV4FlashMLABackend(AttentionBackend): supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [ "auto", - "bfloat16", "fp8_ds_mla", "fp8", # alias for fp8_ds_mla ]