[Refactor] Define MM data parser in processing info instead of processor itself (#33260)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -40,7 +40,6 @@ from ..parse import (
|
||||
DictEmbeddingItems,
|
||||
EmbeddingItems,
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
)
|
||||
from .context import (
|
||||
BaseProcessingInfo,
|
||||
@@ -990,7 +989,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
self.dummy_inputs = dummy_inputs
|
||||
self.cache = cache
|
||||
|
||||
self.data_parser = self._get_data_parser()
|
||||
if hasattr(self, "_get_data_parser"):
|
||||
logger.warning_once(
|
||||
"BaseMultiModalProcessor._get_data_parser is deprecated "
|
||||
"and will be removed in v0.16."
|
||||
"You should override `info.build_data_parser` instead."
|
||||
)
|
||||
|
||||
self.data_parser = self._get_data_parser() # type: ignore
|
||||
else:
|
||||
self.data_parser = self.info.get_data_parser()
|
||||
|
||||
# Avoid unnecessary recomputation
|
||||
self._supported_mm_limits = self.info.get_supported_mm_limits()
|
||||
@@ -1014,26 +1022,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
) -> MultiModalInputs:
|
||||
return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
|
||||
|
||||
def _get_data_parser(self) -> MultiModalDataParser:
|
||||
"""
|
||||
Construct a parser to preprocess multi-modal data items
|
||||
before passing them to
|
||||
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
|
||||
|
||||
You can support additional modalities by creating a subclass
|
||||
of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
|
||||
that has additional subparsers.
|
||||
"""
|
||||
# Get expected hidden size for embedding validation if mm_embeds enabled
|
||||
# This validates hidden dimensions to prevent vulnerabilities: embeddings
|
||||
# with correct ndim but wrong shape could cause crashes at inference time
|
||||
mm_config = self.info.ctx.model_config.get_multimodal_config()
|
||||
expected_hidden_size = None
|
||||
if mm_config.enable_mm_embeds:
|
||||
expected_hidden_size = self.info.ctx.model_config.get_inputs_embeds_size()
|
||||
|
||||
return MultiModalDataParser(expected_hidden_size=expected_hidden_size)
|
||||
|
||||
def validate_num_items(
|
||||
self,
|
||||
modality: str,
|
||||
|
||||
Reference in New Issue
Block a user