[DSV4][Minor] Fix supported KV cache dtypes (#44892)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-06-16 00:14:51 +00:00
committed by GitHub
parent 3afe659b6b
commit f4359a70f9
3 changed files with 8 additions and 8 deletions
+2 -2
View File
@@ -252,6 +252,6 @@ default on NVIDIA is `FLASHMLA_SPARSE_DSV4`.
| Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | Sparse | MM Prefix | DCP | Attention Types | Compute Cap. |
| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ |
| `FLASHINFER_MLA_SPARSE_DSV4` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
| `FLASHMLA_SPARSE_DSV4` | bf16 | `auto`, `bfloat16`, `fp8_ds_mla`, `fp8` | 256 | 512 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x |
| `FLASHINFER_MLA_SPARSE_DSV4` | bf16 | `auto`, `bfloat16`, `fp8` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
| `FLASHMLA_SPARSE_DSV4` | bf16 | `auto`, `fp8_ds_mla`, `fp8` | 256 | 512 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x |
| `ROCM_FLASHMLA_SPARSE_DSV4` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A |
@@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, ClassVar, cast
import torch
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
from vllm.models.deepseek_v4.common.ops import (
@@ -52,13 +53,13 @@ def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
class DeepseekV4FlashInferMLASparseBackend(DeepseekV4FlashMLABackend):
"""Shares the FlashMLA V4 metadata/cache pipeline; swaps the attention impl.
Inheriting from the FlashMLA V4 backend reuses its
``DeepseekV4FlashMLAMetadata`` builder (which the V4 sparse-index
pipeline needs — the V3.2 FlashInfer builder lacks the ``c128a_*`` fields),
256-token blocks, head_size 512, and the (num_blocks, block_size, 512) cache
shape for non-``fp8_ds_mla`` dtypes.
Inheriting from the FlashMLA V4 backend reuses its ``DeepseekV4FlashMLAMetadata``
builder.
"""
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16", "fp8"]
@staticmethod
def get_name() -> str:
return "FLASHINFER_MLA_SPARSE_DSV4"
-1
View File
@@ -46,7 +46,6 @@ class DeepseekV4FlashMLABackend(AttentionBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"auto",
"bfloat16",
"fp8_ds_mla",
"fp8", # alias for fp8_ds_mla
]