[V1] VLM - enable processor cache by default (#11305)
Signed-off-by: Alexander Matveev <alexm@neuralmagic.com>
This commit is contained in:
+5
-6
@@ -148,9 +148,8 @@ class ModelConfig:
|
||||
HuggingFace config.
|
||||
mm_processor_kwargs: Arguments to be forwarded to the model's processor
|
||||
for multi-modal data, e.g., image processor.
|
||||
mm_cache_preprocessor: If true, then enables caching of the multi-modal
|
||||
preprocessor/mapper. Otherwise, the mapper executes each time, and
|
||||
for better performance consider enabling frontend process.
|
||||
disable_mm_preprocessor_cache: If true, then disables caching of the
|
||||
multi-modal preprocessor/mapper. (not recommended)
|
||||
override_neuron_config: Initialize non default neuron config or
|
||||
override default neuron config that are specific to Neuron devices,
|
||||
this argument will be used to configure the neuron config that
|
||||
@@ -216,7 +215,7 @@ class ModelConfig:
|
||||
config_format: ConfigFormat = ConfigFormat.AUTO,
|
||||
hf_overrides: Optional[HfOverrides] = None,
|
||||
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
|
||||
mm_cache_preprocessor: bool = False,
|
||||
disable_mm_preprocessor_cache: bool = False,
|
||||
override_neuron_config: Optional[Dict[str, Any]] = None,
|
||||
override_pooler_config: Optional["PoolerConfig"] = None,
|
||||
logits_processor_pattern: Optional[str] = None) -> None:
|
||||
@@ -286,7 +285,7 @@ class ModelConfig:
|
||||
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
|
||||
self.use_async_output_proc = use_async_output_proc
|
||||
self.mm_processor_kwargs = mm_processor_kwargs
|
||||
self.mm_cache_preprocessor = mm_cache_preprocessor
|
||||
self.disable_mm_preprocessor_cache = disable_mm_preprocessor_cache
|
||||
|
||||
# Set enforce_eager to False if the value is unset.
|
||||
if self.enforce_eager is None:
|
||||
@@ -3155,7 +3154,7 @@ class VllmConfig:
|
||||
f"enable_prefix_caching={self.cache_config.enable_prefix_caching}, "
|
||||
f"chunked_prefill_enabled={self.scheduler_config.chunked_prefill_enabled}, " # noqa
|
||||
f"use_async_output_proc={self.model_config.use_async_output_proc}, "
|
||||
f"mm_cache_preprocessor={self.model_config.mm_cache_preprocessor!r}, " # noqa
|
||||
f"disable_mm_preprocessor_cache={self.model_config.disable_mm_preprocessor_cache!r}, " # noqa
|
||||
f"mm_processor_kwargs={self.model_config.mm_processor_kwargs}, "
|
||||
f"pooler_config={self.model_config.pooler_config!r}, "
|
||||
f"compilation_config={self.compilation_config!r}")
|
||||
|
||||
@@ -141,7 +141,7 @@ class EngineArgs:
|
||||
tokenizer_pool_extra_config: Optional[Dict[str, Any]] = None
|
||||
limit_mm_per_prompt: Optional[Mapping[str, int]] = None
|
||||
mm_processor_kwargs: Optional[Dict[str, Any]] = None
|
||||
mm_cache_preprocessor: bool = False
|
||||
disable_mm_preprocessor_cache: bool = False
|
||||
enable_lora: bool = False
|
||||
enable_lora_bias: bool = False
|
||||
max_loras: int = 1
|
||||
@@ -606,11 +606,10 @@ class EngineArgs:
|
||||
help=('Overrides for the multimodal input mapping/processing, '
|
||||
'e.g., image processor. For example: {"num_crops": 4}.'))
|
||||
parser.add_argument(
|
||||
'--mm-cache-preprocessor',
|
||||
'--disable-mm-preprocessor-cache',
|
||||
action='store_true',
|
||||
help='If true, then enables caching of the multi-modal '
|
||||
'preprocessor/mapper. Otherwise, the mapper executes each time'
|
||||
', and for better performance consider enabling frontend process.')
|
||||
help='If true, then disables caching of the multi-modal '
|
||||
'preprocessor/mapper. (not recommended)')
|
||||
|
||||
# LoRA related configs
|
||||
parser.add_argument('--enable-lora',
|
||||
@@ -983,7 +982,7 @@ class EngineArgs:
|
||||
use_async_output_proc=not self.disable_async_output_proc,
|
||||
config_format=self.config_format,
|
||||
mm_processor_kwargs=self.mm_processor_kwargs,
|
||||
mm_cache_preprocessor=self.mm_cache_preprocessor,
|
||||
disable_mm_preprocessor_cache=self.disable_mm_preprocessor_cache,
|
||||
override_neuron_config=self.override_neuron_config,
|
||||
override_pooler_config=self.override_pooler_config,
|
||||
logits_processor_pattern=self.logits_processor_pattern)
|
||||
|
||||
@@ -191,7 +191,7 @@ def generate_block_hash_extra_keys(
|
||||
raise ValueError(
|
||||
"The number of multi-modal positions and hashes must match. This "
|
||||
"is likely because you do not enable MM preprocessor hashing. "
|
||||
"Please set mm_cache_preprocessor=True.")
|
||||
"Please set disable_mm_preprocessor_cache=False.")
|
||||
|
||||
# Note that we assume mm_positions is sorted by offset.
|
||||
# We do not need to check all mm inputs if the start token index is out of
|
||||
|
||||
@@ -43,7 +43,7 @@ class MMInputMapperClient:
|
||||
self.mm_registry.init_mm_limits_per_prompt(model_config)
|
||||
|
||||
# Init cache
|
||||
self.use_cache = model_config.mm_cache_preprocessor
|
||||
self.use_cache = not model_config.disable_mm_preprocessor_cache
|
||||
self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
|
||||
|
||||
# DEBUG: Set to None to disable
|
||||
@@ -119,7 +119,7 @@ class MMInputMapperClient:
|
||||
class MMInputMapperServer:
|
||||
|
||||
def __init__(self, model_config):
|
||||
self.use_cache = model_config.mm_cache_preprocessor
|
||||
self.use_cache = not model_config.disable_mm_preprocessor_cache
|
||||
self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
|
||||
|
||||
def process_inputs(
|
||||
@@ -151,12 +151,26 @@ class MMHasher:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def hash(self, prompt: PromptType) -> Optional[List[str]]:
|
||||
def hash_mm_data(
|
||||
self,
|
||||
mm_data: Optional[MultiModalDataDict]) -> Optional[List[str]]:
|
||||
if mm_data is None:
|
||||
return None
|
||||
|
||||
image_inputs = mm_data['image']
|
||||
|
||||
return self.hash_images(image_inputs)
|
||||
|
||||
def hash_prompt(self, prompt: PromptType) -> Optional[List[str]]:
|
||||
if "multi_modal_data" not in prompt:
|
||||
return None
|
||||
|
||||
mm_data = prompt["multi_modal_data"]
|
||||
image_inputs = mm_data["image"]
|
||||
|
||||
return self.hash_images(image_inputs)
|
||||
|
||||
def hash_images(self, image_inputs) -> Optional[List[str]]:
|
||||
if not isinstance(image_inputs, list):
|
||||
image_inputs = [image_inputs]
|
||||
assert len(image_inputs) > 0
|
||||
|
||||
@@ -46,7 +46,7 @@ class Processor:
|
||||
self.mm_input_mapper_client = MMInputMapperClient(model_config)
|
||||
|
||||
# Multi-modal hasher (for images)
|
||||
self.use_hash = model_config.mm_cache_preprocessor or \
|
||||
self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
|
||||
cache_config.enable_prefix_caching
|
||||
self.mm_hasher = MMHasher()
|
||||
|
||||
@@ -80,7 +80,7 @@ class Processor:
|
||||
# Compute MM hashes (if enabled)
|
||||
mm_hashes = None
|
||||
if self.use_hash:
|
||||
mm_hashes = self.mm_hasher.hash(prompt)
|
||||
mm_hashes = self.mm_hasher.hash_prompt(prompt)
|
||||
|
||||
# Process inputs.
|
||||
preprocessed_inputs = self.input_preprocessor.preprocess(
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
|
||||
LayerBlockType, cdiv, is_pin_memory_available)
|
||||
from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
|
||||
FlashAttentionMetadata)
|
||||
from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
|
||||
from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
@@ -79,8 +79,14 @@ class GPUModelRunner:
|
||||
# Multi-modal data support
|
||||
self.input_registry = INPUT_REGISTRY
|
||||
self.mm_registry = MULTIMODAL_REGISTRY
|
||||
# NOTE: mm_input_mapper is only used for memory profiling.
|
||||
self.mm_input_mapper = MMInputMapperClient(self.model_config)
|
||||
|
||||
# NOTE: mm_input_mapper_client and mm_hasher are only used for memory
|
||||
# profiling.
|
||||
self.mm_input_mapper_client = MMInputMapperClient(self.model_config)
|
||||
self.mm_hasher = MMHasher()
|
||||
self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
|
||||
cache_config.enable_prefix_caching
|
||||
|
||||
self.max_num_encoder_input_tokens = self.scheduler_config.max_num_encoder_input_tokens # noqa: E501
|
||||
self.encoder_cache_size = self.scheduler_config.encoder_cache_size
|
||||
|
||||
@@ -628,9 +634,15 @@ class GPUModelRunner:
|
||||
mm_registry=self.mm_registry,
|
||||
)
|
||||
dummy_mm_data = dummy_request_data.multi_modal_data
|
||||
dummy_mm_kwargs, _ = self.mm_input_mapper.process_inputs(
|
||||
|
||||
# Compute MM hashes (if enabled)
|
||||
mm_hashes = None
|
||||
if self.use_hash:
|
||||
mm_hashes = self.mm_hasher.hash_mm_data(dummy_mm_data)
|
||||
|
||||
dummy_mm_kwargs = self.mm_input_mapper_client.process_inputs(
|
||||
mm_data=dummy_mm_data,
|
||||
mm_hashes=None,
|
||||
mm_hashes=mm_hashes,
|
||||
mm_processor_kwargs=None,
|
||||
precomputed_mm_inputs=None)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user