[Frontend] Use engine argument to control MM cache size (#22441)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-08-07 09:47:10 -07:00
committed by GitHub
parent 8c9da6be22
commit 139d155781
13 changed files with 101 additions and 47 deletions
+3 -4
View File
@@ -138,13 +138,13 @@ def run_multi_api_server(args: argparse.Namespace):
num_api_servers = args.api_server_count
assert num_api_servers > 0
orig_disable_mm_preprocessor_cache = args.disable_mm_preprocessor_cache
orig_mm_processor_cache_gb = args.mm_processor_cache_gb
if num_api_servers > 1:
setup_multiprocess_prometheus()
# Not compatible with API server scale-out
args.disable_mm_preprocessor_cache = True
args.mm_processor_cache_gb = 0
listen_address, sock = setup_server(args)
@@ -161,8 +161,7 @@ def run_multi_api_server(args: argparse.Namespace):
raise ValueError("VLLM_ALLOW_RUNTIME_LORA_UPDATING cannot be used "
"with api_server_count > 1")
if model_config.is_multimodal_model and not (
orig_disable_mm_preprocessor_cache):
if model_config.is_multimodal_model and orig_mm_processor_cache_gb > 0:
logger.warning("Multi-modal processor cache is disabled because "
"it is not compatible with `api_server_count > 1`.")