[Refactor] Define MM data parser in processing info instead of processor itself (#33260)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-01-29 13:55:17 +08:00
committed by GitHub
parent 07ea184f00
commit 51550179fc
34 changed files with 399 additions and 347 deletions
+30
View File
@@ -17,6 +17,7 @@ import torch
from typing_extensions import TypeVar
from vllm.logger import init_logger
from vllm.multimodal.parse import MultiModalDataParser
from vllm.tokenizers import TokenizerLike
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
@@ -569,6 +570,35 @@ class BaseProcessingInfo:
"""
return self.ctx.get_hf_processor(**kwargs)
def _get_expected_hidden_size(self) -> int | None:
"""
Get expected hidden size for embedding validation if `mm_embeds` are enabled.
This validates hidden dimensions to prevent a vulnerability where embeddings
with correct `ndim` but wrong `shape` could cause crashes at inference time.
"""
model_config = self.ctx.model_config
mm_config = model_config.get_multimodal_config()
if mm_config.enable_mm_embeds:
return model_config.get_inputs_embeds_size()
return None
def get_data_parser(self) -> MultiModalDataParser:
"""
Constructs a parser to preprocess multi-modal data items
before passing them to
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
You can support additional modalities by creating a subclass
of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
that has additional subparsers.
"""
return MultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
)
@property
def skip_prompt_length_check(self) -> bool:
return False
+10 -22
View File
@@ -40,7 +40,6 @@ from ..parse import (
DictEmbeddingItems,
EmbeddingItems,
MultiModalDataItems,
MultiModalDataParser,
)
from .context import (
BaseProcessingInfo,
@@ -990,7 +989,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self.dummy_inputs = dummy_inputs
self.cache = cache
self.data_parser = self._get_data_parser()
if hasattr(self, "_get_data_parser"):
logger.warning_once(
"BaseMultiModalProcessor._get_data_parser is deprecated "
"and will be removed in v0.16."
"You should override `info.build_data_parser` instead."
)
self.data_parser = self._get_data_parser() # type: ignore
else:
self.data_parser = self.info.get_data_parser()
# Avoid unnecessary recomputation
self._supported_mm_limits = self.info.get_supported_mm_limits()
@@ -1014,26 +1022,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
) -> MultiModalInputs:
return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
def _get_data_parser(self) -> MultiModalDataParser:
"""
Construct a parser to preprocess multi-modal data items
before passing them to
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
You can support additional modalities by creating a subclass
of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
that has additional subparsers.
"""
# Get expected hidden size for embedding validation if mm_embeds enabled
# This validates hidden dimensions to prevent vulnerabilities: embeddings
# with correct ndim but wrong shape could cause crashes at inference time
mm_config = self.info.ctx.model_config.get_multimodal_config()
expected_hidden_size = None
if mm_config.enable_mm_embeds:
expected_hidden_size = self.info.ctx.model_config.get_inputs_embeds_size()
return MultiModalDataParser(expected_hidden_size=expected_hidden_size)
def validate_num_items(
self,
modality: str,