diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 4f729f6a06..032f849841 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -276,6 +276,7 @@ from vllm.v1.kv_cache_interface import ( AttentionSpec, KVCacheSpec, MLAAttentionSpec, + get_kv_quant_mode, ) logger = init_logger(__name__) @@ -1025,6 +1026,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): head_size=self.head_size, dtype=kv_cache_dtype, cache_dtype_str=vllm_config.cache_config.cache_dtype, + kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), ) def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):