[1/N][Cleanup] Standardize on use of is_quantized_kv_cache (#38659)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -8,6 +8,7 @@ from pydantic import Field, SkipValidation, field_validator, model_validator
|
||||
|
||||
from vllm.config.utils import config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -236,7 +237,7 @@ class CacheConfig:
|
||||
@field_validator("cache_dtype", mode="after")
|
||||
@classmethod
|
||||
def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType:
|
||||
if cache_dtype.startswith("fp8"):
|
||||
if is_quantized_kv_cache(cache_dtype):
|
||||
logger.info(
|
||||
"Using fp8 data type to store kv cache. It reduces the GPU "
|
||||
"memory footprint and boosts the performance. "
|
||||
|
||||
Reference in New Issue
Block a user