[Attention][DSA] Enable W4A16 DSA (#51724)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
@@ -45,6 +45,7 @@ CacheDType = Literal[
|
||||
"fp8_e5m2",
|
||||
"fp8_inc",
|
||||
"fp8_ds_mla",
|
||||
"nvfp4_ds_mla",
|
||||
"turboquant_k8v4",
|
||||
"turboquant_4bit_nc",
|
||||
"turboquant_k3v4_nc",
|
||||
|
||||
+5
-1
@@ -2821,14 +2821,18 @@ class VllmConfig:
|
||||
def validate_nvfp4_kv_cache_with_mla(self) -> "VllmConfig":
|
||||
if self.model_config is None:
|
||||
return self
|
||||
# The ds_mla layouts are MLA-only by construction; the plain nvfp4
|
||||
# layout (head_size//2 + head_size//16) does not apply to MLA.
|
||||
if (
|
||||
self.cache_config.cache_dtype.startswith("nvfp4")
|
||||
and not self.cache_config.cache_dtype.endswith("_ds_mla")
|
||||
and self.model_config.use_mla
|
||||
):
|
||||
raise ValueError(
|
||||
"nvfp4 KV cache is not supported with MLA (Multi-head Latent "
|
||||
"Attention) backends. Please use a different --kv-cache-dtype "
|
||||
"(e.g., 'fp8' or 'auto') for MLA models such as DeepSeek."
|
||||
"(e.g., 'fp8', 'auto', or 'nvfp4_ds_mla' with a sparse MLA "
|
||||
"backend) for MLA models such as DeepSeek."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
Reference in New Issue
Block a user