[MRV2] Add encoder cache profiling implementation (#47985)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-07-20 20:18:14 -07:00
committed by GitHub
parent 94ed0bf4e0
commit ea0e9c8f2e
4 changed files with 93 additions and 2 deletions
+28 -1
View File
@@ -4,6 +4,7 @@ from collections.abc import Mapping
from vllm.config import ModelConfig, VllmConfig
from vllm.logger import init_logger
from vllm.multimodal.inputs import MultiModalKwargsItem
from vllm.multimodal.processing import BaseMultiModalProcessor
from vllm.multimodal.registry import MultiModalRegistry
from vllm.utils.torch_utils import set_default_torch_num_threads
@@ -48,6 +49,8 @@ class MultiModalBudget:
self,
vllm_config: VllmConfig,
mm_registry: MultiModalRegistry,
*,
enable_cache: bool = True,
) -> None:
super().__init__()
@@ -58,7 +61,11 @@ class MultiModalBudget:
self.max_num_reqs = scheduler_config.max_num_seqs
with set_default_torch_num_threads(): # Avoid hang during startup
cache = mm_registry.processor_only_cache_from_config(vllm_config)
cache = (
mm_registry.processor_only_cache_from_config(vllm_config)
if enable_cache
else None
)
processor = mm_registry.create_processor(model_config, cache=cache)
self.cache = cache
@@ -191,3 +198,23 @@ class MultiModalBudget:
def reset_cache(self) -> None:
if self.cache is not None:
self.cache.clear_cache()
def get_dummy_encoder_profile_inputs(
mm_registry: MultiModalRegistry,
budget: MultiModalBudget,
) -> list[tuple[str, MultiModalKwargsItem]]:
if budget.get_encoder_budget() <= 0 or not budget.mm_max_toks_per_item:
return []
modality = budget.get_modality_with_max_tokens()
max_items_per_batch = budget.mm_max_items_per_batch[modality]
dummy_mm_inputs = mm_registry.get_dummy_mm_inputs(
budget.model_config,
mm_counts={modality: 1},
processor=budget.processor,
)
dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
assert dummy_mm_item is not None, "Dummy item should be generated"
return [(modality, dummy_mm_item)] * max_items_per_batch