[Bugfix] Set kv_quant_mode on the generic MLA KV-cache spec (#48379)
Signed-off-by: Mikhail Kostryukov <mike@triptrack.net> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
parent
37aa52821d
commit
6472131298
@@ -276,6 +276,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -1025,6 +1026,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
head_size=self.head_size,
|
||||
dtype=kv_cache_dtype,
|
||||
cache_dtype_str=vllm_config.cache_config.cache_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
)
|
||||
|
||||
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):
|
||||
|
||||
Reference in New Issue
Block a user