[DSV4][Minor] Fix supported KV cache dtypes (#44892)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -252,6 +252,6 @@ default on NVIDIA is `FLASHMLA_SPARSE_DSV4`.
|
||||
|
||||
| Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | Sparse | MM Prefix | DCP | Attention Types | Compute Cap. |
|
||||
| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ |
|
||||
| `FLASHINFER_MLA_SPARSE_DSV4` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
|
||||
| `FLASHMLA_SPARSE_DSV4` | bf16 | `auto`, `bfloat16`, `fp8_ds_mla`, `fp8` | 256 | 512 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x |
|
||||
| `FLASHINFER_MLA_SPARSE_DSV4` | bf16 | `auto`, `bfloat16`, `fp8` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
|
||||
| `FLASHMLA_SPARSE_DSV4` | bf16 | `auto`, `fp8_ds_mla`, `fp8` | 256 | 512 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x |
|
||||
| `ROCM_FLASHMLA_SPARSE_DSV4` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A |
|
||||
|
||||
@@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, ClassVar, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.common.ops import (
|
||||
@@ -52,13 +53,13 @@ def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
|
||||
class DeepseekV4FlashInferMLASparseBackend(DeepseekV4FlashMLABackend):
|
||||
"""Shares the FlashMLA V4 metadata/cache pipeline; swaps the attention impl.
|
||||
|
||||
Inheriting from the FlashMLA V4 backend reuses its
|
||||
``DeepseekV4FlashMLAMetadata`` builder (which the V4 sparse-index
|
||||
pipeline needs — the V3.2 FlashInfer builder lacks the ``c128a_*`` fields),
|
||||
256-token blocks, head_size 512, and the (num_blocks, block_size, 512) cache
|
||||
shape for non-``fp8_ds_mla`` dtypes.
|
||||
Inheriting from the FlashMLA V4 backend reuses its ``DeepseekV4FlashMLAMetadata``
|
||||
builder.
|
||||
"""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16", "fp8"]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHINFER_MLA_SPARSE_DSV4"
|
||||
|
||||
@@ -46,7 +46,6 @@ class DeepseekV4FlashMLABackend(AttentionBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"bfloat16",
|
||||
"fp8_ds_mla",
|
||||
"fp8", # alias for fp8_ds_mla
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user