Fix DeepSeek V4 FlashMLA auto KV cache dtype (#45091)

Signed-off-by: elehayym <huangxh1618@gmail.com>
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
Co-authored-by: elehayym <huangxh1618@gmail.com>
Co-authored-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
This commit is contained in:
elehayym
2026-09-03 19:29:14 +00:00
committed by GitHub
co-authored by elehayym Kevin Luu
parent 2db1c4dc31
commit fc8f10792c
+9 -4
View File
@@ -102,10 +102,15 @@ def _resolve_dsv4_kv_cache_dtype(
"""
if use_fp8_ds_mla_layout:
# fp8_ds_mla block format: UE8M0 block-scaled fp8 packed as uint8.
assert kv_cache_dtype.startswith("fp8"), (
f"DeepseekV4 fp8_ds_mla layout only supports fp8 kv-cache, "
f"got {kv_cache_dtype}"
)
if kv_cache_dtype == "auto":
kv_cache_dtype = "fp8"
if not kv_cache_dtype.startswith("fp8"):
raise ValueError(
"DeepseekV4 fp8_ds_mla layout only supports fp8 "
f"kv-cache, got {kv_cache_dtype}. Please set "
"`--kv-cache-dtype fp8` or select a backend that supports "
"bfloat16 KV cache."
)
if kv_cache_dtype != "fp8_ds_mla":
if cache_config is not None:
cache_config.cache_dtype = "fp8_ds_mla"