[Renderer] Move InputPreprocessor into Renderer (1/2) (#34510)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
@@ -33,9 +33,9 @@ from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
|
||||
from vllm.utils.jsontree import json_iter_leaves
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.metrics.stats import MultiModalCacheStats
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -60,8 +60,6 @@ class InputProcessor:
|
||||
self.generation_config_fields = model_config.try_get_generation_config()
|
||||
|
||||
self.renderer = renderer or renderer_from_config(vllm_config)
|
||||
self.mm_registry = mm_registry
|
||||
self.mm_processor_cache = mm_registry.processor_cache_from_config(vllm_config)
|
||||
|
||||
self.supports_mm_inputs = mm_registry.supports_multimodal_inputs(model_config)
|
||||
self.mm_encoder_cache_size = 0
|
||||
@@ -78,7 +76,6 @@ class InputProcessor:
|
||||
vllm_config,
|
||||
renderer=renderer,
|
||||
mm_registry=mm_registry,
|
||||
mm_processor_cache=self.mm_processor_cache,
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -136,7 +133,7 @@ class InputProcessor:
|
||||
)
|
||||
|
||||
def _parse_mm_items(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:
|
||||
mm_processor = self.input_preprocessor._get_mm_processor()
|
||||
mm_processor = self.renderer.get_mm_processor()
|
||||
return mm_processor.info.parse_mm_data(mm_data)
|
||||
|
||||
def _validate_singleton_mm_uuids(self, prompt: SingletonPrompt) -> None:
|
||||
@@ -415,6 +412,15 @@ class InputProcessor:
|
||||
decoder_mm_positions = decoder_inputs["mm_placeholders"]
|
||||
decoder_mm_hashes = decoder_inputs["mm_hashes"]
|
||||
|
||||
if not all(
|
||||
isinstance(leaf, str) for leaf in json_iter_leaves(decoder_mm_hashes)
|
||||
):
|
||||
raise ValueError(
|
||||
f"mm_hashes must contain only strings, got: {decoder_mm_hashes}. "
|
||||
"This is likely due to an incorrect custom implementation of "
|
||||
"MultiModalProcessor.apply method."
|
||||
)
|
||||
|
||||
# Merge and flatten multimodal placeholders, hashes and inputs
|
||||
# from dictionaries to lists, and sort them by each item's position
|
||||
# in the input sequence.
|
||||
@@ -562,13 +568,3 @@ class InputProcessor:
|
||||
self._validate_model_input(encoder_inputs, prompt_type="encoder")
|
||||
|
||||
self._validate_model_input(decoder_inputs, prompt_type="decoder")
|
||||
|
||||
def stat_mm_cache(self) -> MultiModalCacheStats | None:
|
||||
return self.input_preprocessor.stat_mm_cache()
|
||||
|
||||
def clear_mm_cache(self) -> None:
|
||||
self.input_preprocessor.clear_mm_cache()
|
||||
|
||||
def close(self) -> None:
|
||||
if self.mm_processor_cache is not None:
|
||||
self.mm_processor_cache.close()
|
||||
|
||||
Reference in New Issue
Block a user