[Mamba] enable prefix cache by default (#50991)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7b50d2c0bc
commit
f9c74b4b9c
@@ -595,10 +595,8 @@ class MambaModelConfig(VerifyAndUpdateConfig):
|
||||
|
||||
if cache_config.enable_prefix_caching:
|
||||
if cache_config.mamba_cache_mode == "none":
|
||||
cache_config.mamba_cache_mode = (
|
||||
"all" if model_config.supports_mamba_prefix_caching else "align"
|
||||
)
|
||||
logger.warning(
|
||||
cache_config.mamba_cache_mode = "align"
|
||||
logger.info(
|
||||
"Mamba cache mode is set to '%s' for %s by default "
|
||||
"when prefix caching is enabled",
|
||||
cache_config.mamba_cache_mode,
|
||||
@@ -618,13 +616,6 @@ class MambaModelConfig(VerifyAndUpdateConfig):
|
||||
assert vllm_config.scheduler_config.enable_chunked_prefill, (
|
||||
"Chunked prefill is required for mamba cache mode 'align'."
|
||||
)
|
||||
logger.info(
|
||||
"Warning: Prefix caching in Mamba cache '%s' "
|
||||
"mode is currently enabled. "
|
||||
"Its support for Mamba layers is experimental. "
|
||||
"Please report any issues you may observe.",
|
||||
cache_config.mamba_cache_mode,
|
||||
)
|
||||
# By default, mamba block size will be set to max_model_len (see
|
||||
# below). When enabling prefix caching, we align mamba block size
|
||||
# to the block size as the basic granularity for prefix caching.
|
||||
|
||||
Reference in New Issue
Block a user