[Refactor] Define MM data parser in processing info instead of processor itself (#33260)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-01-29 13:55:17 +08:00
committed by GitHub
parent 07ea184f00
commit 51550179fc
34 changed files with 399 additions and 347 deletions
+10 -22
View File
@@ -40,7 +40,6 @@ from ..parse import (
DictEmbeddingItems,
EmbeddingItems,
MultiModalDataItems,
MultiModalDataParser,
)
from .context import (
BaseProcessingInfo,
@@ -990,7 +989,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self.dummy_inputs = dummy_inputs
self.cache = cache
self.data_parser = self._get_data_parser()
if hasattr(self, "_get_data_parser"):
logger.warning_once(
"BaseMultiModalProcessor._get_data_parser is deprecated "
"and will be removed in v0.16."
"You should override `info.build_data_parser` instead."
)
self.data_parser = self._get_data_parser() # type: ignore
else:
self.data_parser = self.info.get_data_parser()
# Avoid unnecessary recomputation
self._supported_mm_limits = self.info.get_supported_mm_limits()
@@ -1014,26 +1022,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
) -> MultiModalInputs:
return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
def _get_data_parser(self) -> MultiModalDataParser:
"""
Construct a parser to preprocess multi-modal data items
before passing them to
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
You can support additional modalities by creating a subclass
of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
that has additional subparsers.
"""
# Get expected hidden size for embedding validation if mm_embeds enabled
# This validates hidden dimensions to prevent vulnerabilities: embeddings
# with correct ndim but wrong shape could cause crashes at inference time
mm_config = self.info.ctx.model_config.get_multimodal_config()
expected_hidden_size = None
if mm_config.enable_mm_embeds:
expected_hidden_size = self.info.ctx.model_config.get_inputs_embeds_size()
return MultiModalDataParser(expected_hidden_size=expected_hidden_size)
def validate_num_items(
self,
modality: str,