[Bugfix] Disable prefix caching for encoder-only before model config hooks (#58287)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
+13
-10
@@ -1340,6 +1340,19 @@ class VllmConfig:
|
||||
|
||||
self._resolve_mm_encoder_only()
|
||||
|
||||
if self.is_mm_encoder_only and self.cache_config.enable_prefix_caching:
|
||||
# Such an instance publishes encoder embeddings and runs no language
|
||||
# model, so it holds no KV cache for prefix caching to reuse and its
|
||||
# coordinator would have no group to manage. Disable before
|
||||
# `try_verify_and_update_config` so model config hooks (e.g. the
|
||||
# hybrid mamba hook setting `mamba_block_size`) already see prefix
|
||||
# caching as disabled.
|
||||
logger.info(
|
||||
"Disabling prefix caching: this instance runs the "
|
||||
"multi-modal encoder only."
|
||||
)
|
||||
self.cache_config.enable_prefix_caching = False
|
||||
|
||||
if self.performance_mode != "balanced":
|
||||
logger.info_once("Performance mode set to '%s'.", self.performance_mode)
|
||||
|
||||
@@ -2089,16 +2102,6 @@ class VllmConfig:
|
||||
self._post_init_kv_transfer_config()
|
||||
self._verify_aux_output_compatibility()
|
||||
|
||||
if self.is_mm_encoder_only and self.cache_config.enable_prefix_caching:
|
||||
# Such an instance publishes encoder embeddings and runs no language
|
||||
# model, so it holds no KV cache for prefix caching to reuse and its
|
||||
# coordinator would have no group to manage.
|
||||
logger.info(
|
||||
"Disabling prefix caching: this instance runs the "
|
||||
"multi-modal encoder only."
|
||||
)
|
||||
self.cache_config.enable_prefix_caching = False
|
||||
|
||||
# Hybrid KV cache manager (HMA) runtime rules:
|
||||
# - Explicit enable (--no-disable-kv-cache-manager): error if runtime
|
||||
# disables it
|
||||
|
||||
Reference in New Issue
Block a user