Fix DeepSeek V4 FlashMLA auto KV cache dtype (#45091)
Signed-off-by: elehayym <huangxh1618@gmail.com> Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> Co-authored-by: elehayym <huangxh1618@gmail.com> Co-authored-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
This commit is contained in:
co-authored by
elehayym
Kevin Luu
parent
2db1c4dc31
commit
fc8f10792c
@@ -102,10 +102,15 @@ def _resolve_dsv4_kv_cache_dtype(
|
||||
"""
|
||||
if use_fp8_ds_mla_layout:
|
||||
# fp8_ds_mla block format: UE8M0 block-scaled fp8 packed as uint8.
|
||||
assert kv_cache_dtype.startswith("fp8"), (
|
||||
f"DeepseekV4 fp8_ds_mla layout only supports fp8 kv-cache, "
|
||||
f"got {kv_cache_dtype}"
|
||||
)
|
||||
if kv_cache_dtype == "auto":
|
||||
kv_cache_dtype = "fp8"
|
||||
if not kv_cache_dtype.startswith("fp8"):
|
||||
raise ValueError(
|
||||
"DeepseekV4 fp8_ds_mla layout only supports fp8 "
|
||||
f"kv-cache, got {kv_cache_dtype}. Please set "
|
||||
"`--kv-cache-dtype fp8` or select a backend that supports "
|
||||
"bfloat16 KV cache."
|
||||
)
|
||||
if kv_cache_dtype != "fp8_ds_mla":
|
||||
if cache_config is not None:
|
||||
cache_config.cache_dtype = "fp8_ds_mla"
|
||||
|
||||
Reference in New Issue
Block a user