[1/N][Cleanup] Standardize on use of is_quantized_kv_cache (#38659)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -16,7 +16,7 @@ import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import is_quantized_kv_cache
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .interface import CpuArchEnum, Platform, PlatformEnum
|
||||
@@ -183,7 +183,7 @@ class CpuPlatform(Platform):
|
||||
"backend is not compatible with FP8 KV cache."
|
||||
)
|
||||
|
||||
if cache_config.cache_dtype.startswith("fp8"):
|
||||
if is_quantized_kv_cache(cache_config.cache_dtype):
|
||||
logger.warning(
|
||||
"CPU backend doesn't support KV cache quantization fallback to auto."
|
||||
)
|
||||
|
||||
@@ -23,6 +23,7 @@ import vllm._C_stable_libtorch # noqa
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import import_pynvml
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .interface import DeviceCapability, Platform, PlatformEnum
|
||||
@@ -87,7 +88,7 @@ def _get_backend_priorities(
|
||||
# Sparse MLA backend priorities
|
||||
# See https://github.com/vllm-project/vllm/issues/35807 for
|
||||
# benchmark results
|
||||
if kv_cache_dtype is not None and kv_cache_dtype.startswith("fp8"):
|
||||
if kv_cache_dtype is not None and is_quantized_kv_cache(kv_cache_dtype):
|
||||
# Prefer FlashInfer for fp8 kv cache
|
||||
sparse_backends = [
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE,
|
||||
|
||||
Reference in New Issue
Block a user