From fc8f10792c59011447b56b41d4c8b1048673e939 Mon Sep 17 00:00:00 2001 From: elehayym <52448798+Yuzu23@users.noreply.github.com> Date: Fri, 4 Sep 2026 03:29:14 +0800 Subject: [PATCH] Fix DeepSeek V4 FlashMLA auto KV cache dtype (#45091) Signed-off-by: elehayym Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> Co-authored-by: elehayym Co-authored-by: Kevin Luu <51931015+khluu@users.noreply.github.com> --- vllm/models/deepseek_v4/attention.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index ce9fe06a2f..d62e4d5bf5 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -102,10 +102,15 @@ def _resolve_dsv4_kv_cache_dtype( """ if use_fp8_ds_mla_layout: # fp8_ds_mla block format: UE8M0 block-scaled fp8 packed as uint8. - assert kv_cache_dtype.startswith("fp8"), ( - f"DeepseekV4 fp8_ds_mla layout only supports fp8 kv-cache, " - f"got {kv_cache_dtype}" - ) + if kv_cache_dtype == "auto": + kv_cache_dtype = "fp8" + if not kv_cache_dtype.startswith("fp8"): + raise ValueError( + "DeepseekV4 fp8_ds_mla layout only supports fp8 " + f"kv-cache, got {kv_cache_dtype}. Please set " + "`--kv-cache-dtype fp8` or select a backend that supports " + "bfloat16 KV cache." + ) if kv_cache_dtype != "fp8_ds_mla": if cache_config is not None: cache_config.cache_dtype = "fp8_ds_mla"