[Frontend] Use engine argument to control MM cache size (#22441)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+32
-11
@@ -443,8 +443,15 @@ class ModelConfig:
|
||||
from `AutoProcessor.from_pretrained`. The available overrides depend on the
|
||||
model that is being run. For example, for Phi-3-Vision: `{"num_crops": 4}`.
|
||||
"""
|
||||
disable_mm_preprocessor_cache: bool = False
|
||||
"""If `True`, disable caching of the multi-modal processor."""
|
||||
mm_processor_cache_gb: int = 4
|
||||
"""The size (in GiB) of the multi-modal processor cache, which is used to
|
||||
avoid re-processing past multi-modal inputs.
|
||||
|
||||
This cache is duplicated for each API process and engine core process,
|
||||
resulting in a total memory usage of
|
||||
`mm_processor_cache_gb * (api_server_count + data_parallel_size)`.
|
||||
|
||||
Set to `0` to disable this cache completely (not recommended)."""
|
||||
override_neuron_config: dict[str, Any] = field(default_factory=dict)
|
||||
"""Initialize non-default neuron config or override default neuron config
|
||||
that are specific to Neuron devices, this argument will be used to
|
||||
@@ -881,17 +888,16 @@ class ModelConfig:
|
||||
limit_per_prompt=self.limit_mm_per_prompt,
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
mm_processor_kwargs=self.mm_processor_kwargs,
|
||||
disable_mm_preprocessor_cache=self.
|
||||
disable_mm_preprocessor_cache,
|
||||
mm_processor_cache_gb=self.mm_processor_cache_gb,
|
||||
interleave_mm_strings=self.interleave_mm_strings)
|
||||
|
||||
return None
|
||||
|
||||
def set_disable_mm_preprocessor_cache(self, value: bool) -> None:
|
||||
def set_mm_processor_cache_gb(self, value: int) -> None:
|
||||
mm_config = self.get_multimodal_config()
|
||||
|
||||
self.disable_mm_preprocessor_cache = value
|
||||
mm_config.disable_mm_preprocessor_cache = value
|
||||
self.mm_processor_cache_gb = value
|
||||
mm_config.mm_processor_cache_gb = value
|
||||
|
||||
def _get_encoder_config(self):
|
||||
return get_sentence_transformer_tokenizer_config(
|
||||
@@ -1698,7 +1704,16 @@ class ModelConfig:
|
||||
if mm_config is None:
|
||||
return False
|
||||
|
||||
return not mm_config.disable_mm_preprocessor_cache
|
||||
return mm_config.mm_processor_cache_gb > 0
|
||||
|
||||
@property
|
||||
def enable_mm_processor_cache(self) -> bool:
|
||||
"""Whether the multi-modal processor cache should be enabled."""
|
||||
mm_config = self.multimodal_config
|
||||
if mm_config is None:
|
||||
return False
|
||||
|
||||
return mm_config.mm_processor_cache_gb > 0
|
||||
|
||||
@property
|
||||
def enable_mm_input_cache(self) -> bool:
|
||||
@@ -1707,7 +1722,7 @@ class ModelConfig:
|
||||
if mm_config is None:
|
||||
return False
|
||||
|
||||
return not mm_config.disable_mm_preprocessor_cache
|
||||
return mm_config.mm_processor_cache_gb > 0
|
||||
|
||||
def get_mm_input_cache_gb(self) -> int:
|
||||
mm_config = self.multimodal_config
|
||||
@@ -3391,9 +3406,15 @@ class MultiModalConfig:
|
||||
`{"num_crops": 4}`.
|
||||
"""
|
||||
|
||||
disable_mm_preprocessor_cache: bool = False
|
||||
mm_processor_cache_gb: int = 4
|
||||
"""
|
||||
If `True`, disable caching of the multi-modal processor.
|
||||
The size (in GiB) of the multi-modal processor cache, which is used to
|
||||
|
||||
This cache is duplicated for each API process and engine core process,
|
||||
resulting in a total memory usage of
|
||||
`mm_processor_cache_gb * (api_server_count + data_parallel_size)`.
|
||||
|
||||
Set to `0` to disable this cache completely (not recommended).
|
||||
"""
|
||||
|
||||
interleave_mm_strings: bool = False
|
||||
|
||||
@@ -358,8 +358,8 @@ class EngineArgs:
|
||||
"media_io_kwargs")
|
||||
mm_processor_kwargs: Optional[Dict[str, Any]] = \
|
||||
MultiModalConfig.mm_processor_kwargs
|
||||
disable_mm_preprocessor_cache: bool = \
|
||||
MultiModalConfig.disable_mm_preprocessor_cache
|
||||
disable_mm_preprocessor_cache: bool = False # DEPRECATED
|
||||
mm_processor_cache_gb: int = MultiModalConfig.mm_processor_cache_gb
|
||||
# LoRA fields
|
||||
enable_lora: bool = False
|
||||
enable_lora_bias: bool = LoRAConfig.bias_enabled
|
||||
@@ -720,8 +720,11 @@ class EngineArgs:
|
||||
"--mm-processor-kwargs",
|
||||
**multimodal_kwargs["mm_processor_kwargs"])
|
||||
multimodal_group.add_argument(
|
||||
"--disable-mm-preprocessor-cache",
|
||||
**multimodal_kwargs["disable_mm_preprocessor_cache"])
|
||||
"--mm-processor-cache-gb",
|
||||
**multimodal_kwargs["mm_processor_cache_gb"])
|
||||
multimodal_group.add_argument("--disable-mm-preprocessor-cache",
|
||||
type=bool,
|
||||
deprecated=True)
|
||||
multimodal_group.add_argument(
|
||||
"--interleave-mm-strings",
|
||||
**multimodal_kwargs["interleave_mm_strings"])
|
||||
@@ -886,6 +889,23 @@ class EngineArgs:
|
||||
self.model = f"{MODEL_WEIGHTS_S3_BUCKET}/{self.model}"
|
||||
self.load_format = "runai_streamer"
|
||||
|
||||
if self.disable_mm_preprocessor_cache:
|
||||
logger.warning(
|
||||
"`--disable-mm-preprocessor-cache` is deprecated "
|
||||
"and will be removed in v0.13. "
|
||||
"Please use `--mm-processor-cache-gb 0` instead.", )
|
||||
|
||||
self.mm_processor_cache_gb = 0
|
||||
elif envs.VLLM_MM_INPUT_CACHE_GIB != 4:
|
||||
logger.warning(
|
||||
"VLLM_MM_INPUT_CACHE_GIB` is deprecated "
|
||||
"and will be removed in v0.13. "
|
||||
"Please use `--mm-processor-cache-gb %d` instead.",
|
||||
envs.VLLM_MM_INPUT_CACHE_GIB,
|
||||
)
|
||||
|
||||
self.mm_processor_cache_gb = envs.VLLM_MM_INPUT_CACHE_GIB
|
||||
|
||||
return ModelConfig(
|
||||
model=self.model,
|
||||
hf_config_path=self.hf_config_path,
|
||||
@@ -922,7 +942,7 @@ class EngineArgs:
|
||||
use_async_output_proc=not self.disable_async_output_proc,
|
||||
config_format=self.config_format,
|
||||
mm_processor_kwargs=self.mm_processor_kwargs,
|
||||
disable_mm_preprocessor_cache=self.disable_mm_preprocessor_cache,
|
||||
mm_processor_cache_gb=self.mm_processor_cache_gb,
|
||||
override_neuron_config=self.override_neuron_config,
|
||||
override_pooler_config=self.override_pooler_config,
|
||||
logits_processor_pattern=self.logits_processor_pattern,
|
||||
@@ -1234,13 +1254,13 @@ class EngineArgs:
|
||||
dp_supports_mm_processor_cache = (self.data_parallel_size == 1
|
||||
or data_parallel_external_lb)
|
||||
if (not dp_supports_mm_processor_cache
|
||||
and not model_config.disable_mm_preprocessor_cache):
|
||||
and model_config.mm_processor_cache_gb > 0):
|
||||
logger.warning(
|
||||
"Multi-modal processor cache is disabled because "
|
||||
"it is not compatible with data parallelism when "
|
||||
"there does not exist a one-to-one correspondance "
|
||||
"between API and engine core processes.")
|
||||
model_config.set_disable_mm_preprocessor_cache(True)
|
||||
model_config.set_mm_processor_cache_gb(0)
|
||||
|
||||
speculative_config = self.create_speculative_config(
|
||||
target_model_config=model_config,
|
||||
|
||||
@@ -138,13 +138,13 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
num_api_servers = args.api_server_count
|
||||
assert num_api_servers > 0
|
||||
|
||||
orig_disable_mm_preprocessor_cache = args.disable_mm_preprocessor_cache
|
||||
orig_mm_processor_cache_gb = args.mm_processor_cache_gb
|
||||
|
||||
if num_api_servers > 1:
|
||||
setup_multiprocess_prometheus()
|
||||
|
||||
# Not compatible with API server scale-out
|
||||
args.disable_mm_preprocessor_cache = True
|
||||
args.mm_processor_cache_gb = 0
|
||||
|
||||
listen_address, sock = setup_server(args)
|
||||
|
||||
@@ -161,8 +161,7 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
raise ValueError("VLLM_ALLOW_RUNTIME_LORA_UPDATING cannot be used "
|
||||
"with api_server_count > 1")
|
||||
|
||||
if model_config.is_multimodal_model and not (
|
||||
orig_disable_mm_preprocessor_cache):
|
||||
if model_config.is_multimodal_model and orig_mm_processor_cache_gb > 0:
|
||||
logger.warning("Multi-modal processor cache is disabled because "
|
||||
"it is not compatible with `api_server_count > 1`.")
|
||||
|
||||
|
||||
+1
-1
@@ -561,7 +561,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_VIDEO_LOADER_BACKEND":
|
||||
lambda: os.getenv("VLLM_VIDEO_LOADER_BACKEND", "opencv"),
|
||||
|
||||
# Cache size (in GiB per process) for multimodal input cache
|
||||
# [DEPRECATED] Cache size (in GiB per process) for multimodal input cache
|
||||
# Default is 4 GiB per API process + 4 GiB per engine core process
|
||||
"VLLM_MM_INPUT_CACHE_GIB":
|
||||
lambda: int(os.getenv("VLLM_MM_INPUT_CACHE_GIB", "4")),
|
||||
|
||||
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Generic, Optional, Protocol, TypeVar
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.envs import VLLM_MM_INPUT_CACHE_GIB
|
||||
from vllm.inputs import InputProcessingContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.tokenizer import (AnyTokenizer,
|
||||
@@ -96,11 +95,22 @@ class MultiModalRegistry:
|
||||
self._processor_factories = ClassRegistry[nn.Module,
|
||||
_ProcessorFactories]()
|
||||
|
||||
self._processing_cache = ProcessingCache(VLLM_MM_INPUT_CACHE_GIB)
|
||||
self._processor_cache: Optional[ProcessingCache] = None
|
||||
|
||||
def _get_processor_cache(self, model_config: "ModelConfig"):
|
||||
capacity_gb = model_config.mm_processor_cache_gb
|
||||
if capacity_gb is None:
|
||||
return None # Overrides `disable_cache` argument
|
||||
|
||||
if self._processor_cache is None:
|
||||
self._processor_cache = ProcessingCache(capacity_gb)
|
||||
|
||||
return self._processor_cache
|
||||
|
||||
def reset_processor_cache(self) -> bool:
|
||||
"""Reset the multi-modal processing cache."""
|
||||
self._processing_cache.reset()
|
||||
if self._processor_cache:
|
||||
self._processor_cache.reset()
|
||||
|
||||
return True # Success
|
||||
|
||||
@@ -244,14 +254,14 @@ class MultiModalRegistry:
|
||||
if tokenizer is None and not model_config.skip_tokenizer_init:
|
||||
tokenizer = cached_tokenizer_from_config(model_config)
|
||||
if disable_cache is None:
|
||||
mm_config = model_config.get_multimodal_config()
|
||||
disable_cache = mm_config.disable_mm_preprocessor_cache
|
||||
disable_cache = not model_config.enable_mm_processor_cache
|
||||
|
||||
model_cls = self._get_model_cls(model_config)
|
||||
factories = self._processor_factories[model_cls]
|
||||
|
||||
ctx = InputProcessingContext(model_config, tokenizer)
|
||||
cache = None if disable_cache else self._processing_cache
|
||||
cache = None if disable_cache else self._get_processor_cache(
|
||||
model_config)
|
||||
|
||||
return factories.build_processor(ctx, cache=cache)
|
||||
|
||||
|
||||
@@ -430,7 +430,7 @@ def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,
|
||||
raise ValueError(
|
||||
"The number of multi-modal positions and hashes must match. This "
|
||||
"is likely because you did not enable MM hashing. "
|
||||
"Please set `disable_mm_preprocessor_cache=False`.")
|
||||
"Please set `mm_processor_cache_gb > 0`.")
|
||||
|
||||
# Note that we assume mm_positions is sorted by offset.
|
||||
# We do not need to check all mm inputs if the start token index is out of
|
||||
|
||||
Reference in New Issue
Block a user