diff --git a/docs/contributing/model/multimodal.md b/docs/contributing/model/multimodal.md index 951d25587a..d374a5ecbc 100644 --- a/docs/contributing/model/multimodal.md +++ b/docs/contributing/model/multimodal.md @@ -454,17 +454,17 @@ return a schema of the tensors outputted by the HF processor that are related to ??? code ```python - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: # Mistral3Processor requires text corresponding to the images return self.dummy_inputs.get_dummy_text(mm_counts) def _postprocess_hf_mm_data( self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], processed_data: BatchFeature, ) -> BatchFeature: - if not mm_data: + if not hf_data: return processed_data pixel_values = processed_data.get("pixel_values") @@ -486,25 +486,36 @@ return a schema of the tensors outputted by the HF processor that are related to calls the HF processor on the multi-modal data without passing any text. If the HF processor instead requires text corresponding to the multi-modal items, you should override - [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text] + [_get_hf_mm_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_text] to return the dummy text from [BaseDummyInputsBuilder.get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text] like in the example above. If the HF processor expects the multi-modal data - under different keys than those provided by the multi-modal items - (e.g. `audio` instead of `audios`), or requires additional keyword arguments + under different keys than those provided by the multi-modal items, + or requires additional keyword arguments (e.g. `sampling_rate`), you should override - [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data]. + [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs]. If you need to modify the output of the HF processor, you should override [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data]. + Passthrough fields have already been merged into `processed_data` when this + hook is called. + If you need to change how the normalized inputs are passed to the HF + processor (for example, to call a method other than `__call__`, or to use + different kwargs to construct and invoke it), override + [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor]. If the HF processor applies additional transformations to the prompt tokens that are not reflected in its multi-modal outputs (e.g. prepending a BOS token), you should override [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt] to replicate them. - For even more control over how the HF processor is called, you can override + Only processors that require multiple HF calls or manually construct the + complete output should override [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] - directly. + directly. Such implementations must obtain their normalized inputs through + [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs] + and return through + [_finalize_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._finalize_hf_mm_data] + so passthrough fields and model-specific postprocessing are applied consistently. Since `pixel_values` is now a list with one tensor per image, we can override [_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows: @@ -724,7 +735,7 @@ Examples: ### Custom HF processor -Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. It is then applied to the multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor] inside [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]. +Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. The default [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor] applies it to the normalized multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor]. Override `_call_hf_processor` if the custom processor instead exposes a different callable interface. Examples: diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index a2112a2922..202d8d04c1 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -8,9 +8,16 @@ In vLLM's rendering pipeline (see [BaseRenderer][vllm.renderers.base.BaseRendere Since Transformers 5.10, `ProcessorMixin` now allows multi-modal inputs to be passed by themselves. However, certain subclasses (such as `ChameleonProcessor`) and older out-of-tree implementations may still define their own `__call__` method that assumes the presence of text with corresponding placeholder tokens. This causes a problem as we don't have the original text anymore to pass to these processors. -To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text] returns so that [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] passes it to the HF processor together with the multi-modal inputs to obtain the processed multi-modal data. +To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_mm_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_text] returns. [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs] adds this text and `truncation=False` to the normalized HF inputs. The truncation setting is attached only to calls that use dummy text, so unrelated processor calls do not receive a text-specific option. -Similarly, since the multi-modal data extracted by vLLM may not match what a specific HF processor expects, [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] allows each model to adapt the inputs via [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data] (e.g. renaming keys like `audios` to `audio` or injecting extra keyword arguments such as `sampling_rate`) and the outputs via [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data], without having to reimplement the entire method. +HF processing is split into four overridable stages: + +1. [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs] extracts and normalizes processor data, kwargs, and passthrough data. It centrally renames vLLM's `audios` field to the standard HF `audio` keyword; processors that require another spelling can adapt it in an override. +2. [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor] invokes the HF processor. Models can override it when they use a different callable or need separate construction and invocation kwargs. +3. [_finalize_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._finalize_hf_mm_data] merges passthrough fields into the processed output. It lazily creates an empty output when there is no processor data. +4. [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data] performs model-specific output transformations after passthrough fields have been merged. + +This division lets most models customize one stage without reimplementing [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]. Implementations that genuinely require multiple processor calls or manually construct their complete output may still override that method, but must use `_get_hf_mm_inputs` and `_finalize_hf_mm_data` to retain the common input and output behavior. ## Prompt Update Detection diff --git a/vllm/model_executor/models/audioflamingo3.py b/vllm/model_executor/models/audioflamingo3.py index 4832d445d6..36f2569277 100644 --- a/vllm/model_executor/models/audioflamingo3.py +++ b/vllm/model_executor/models/audioflamingo3.py @@ -374,21 +374,9 @@ class AudioFlamingo3MultiModalDataParser(MultiModalDataParser): class AudioFlamingo3MultiModalProcessor( BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo] ): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _preprocess_hf_mm_data( - self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - mm_data = dict(mm_data) - audios = mm_data.pop("audios", None) - if audios is not None: - mm_data["audio"] = audios - - return mm_data, hf_processor_mm_kwargs - def _postprocess_hf_mm_data( self, mm_data: Mapping[str, object], diff --git a/vllm/model_executor/models/bailing_moe_v3_vl.py b/vllm/model_executor/models/bailing_moe_v3_vl.py index c926c49ea8..7cd0568530 100644 --- a/vllm/model_executor/models/bailing_moe_v3_vl.py +++ b/vllm/model_executor/models/bailing_moe_v3_vl.py @@ -119,7 +119,7 @@ class BailingMoeV3VLDummyInputsBuilder( class BailingMoeV3VLMultiModalProcessor( BaseMultiModalProcessor[BailingMoeV3VLProcessingInfo] ): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _apply_prompt_updates( diff --git a/vllm/model_executor/models/cohere2_vision.py b/vllm/model_executor/models/cohere2_vision.py index fbb853beb0..a4748932d6 100644 --- a/vllm/model_executor/models/cohere2_vision.py +++ b/vllm/model_executor/models/cohere2_vision.py @@ -222,7 +222,7 @@ class Cohere2VisionDummyInputsBuilder( class Cohere2VisionMultiModalProcessor( BaseMultiModalProcessor[Cohere2VisionProcessingInfo] ): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _postprocess_hf_mm_data( diff --git a/vllm/model_executor/models/cohere_asr.py b/vllm/model_executor/models/cohere_asr.py index 29205e4ed6..5020cadc99 100644 --- a/vllm/model_executor/models/cohere_asr.py +++ b/vllm/model_executor/models/cohere_asr.py @@ -8,7 +8,7 @@ from typing import Any, ClassVar import torch import torch.nn.functional as F from torch import nn -from transformers import BatchFeature, PretrainedConfig +from transformers import PretrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig @@ -50,7 +50,10 @@ from vllm.multimodal.processing import ( PromptUpdate, TimingContext, ) -from vllm.multimodal.processing.processor import MultiModalProcessingInfo +from vllm.multimodal.processing.processor import ( + HFMultiModalInputs, + MultiModalProcessingResult, +) from vllm.renderers import TokenizeParams from vllm.tokenizers import cached_tokenizer_from_config from vllm.transformers_utils.processors.cohere_asr import ( @@ -1962,42 +1965,29 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing ) -> list[int]: return [0] - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _preprocess_hf_mm_data( + def _get_hf_mm_inputs( self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + mm_items: MultiModalDataItems, + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) - mm_data = dict(mm_data) - mm_data["audio"] = mm_data.pop("audios") - - hf_processor_mm_kwargs = dict( - **hf_processor_mm_kwargs, - sampling_rate=feature_extractor.sampling_rate, + feature_extractor = self.info.get_feature_extractor(**hf_kwargs) + return hf_inputs._replace( + hf_kwargs=dict( + hf_inputs.hf_kwargs, + sampling_rate=feature_extractor.sampling_rate, + ) ) - return mm_data, hf_processor_mm_kwargs - - def _postprocess_hf_mm_data( - self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - processed_data: BatchFeature, - ) -> BatchFeature: - if "labels" in processed_data: - processed_data["input_ids"] = processed_data.pop("labels") - - return processed_data - def _cached_apply_hf_processor( self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> MultiModalProcessingInfo: + ) -> MultiModalProcessingResult: # Dithering injects noise into the extracted features, so the # feature extractor is not a pure function of its input. Since the # processing cache assumes that processor outputs are invariant diff --git a/vllm/model_executor/models/colmodernvbert.py b/vllm/model_executor/models/colmodernvbert.py index 89e55d40e2..0c93e7ef52 100644 --- a/vllm/model_executor/models/colmodernvbert.py +++ b/vllm/model_executor/models/colmodernvbert.py @@ -157,15 +157,14 @@ class ColModernVBertMultiModalProcessor( def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) if not mm_data: - return BatchFeature(dict(passthrough_data)) + return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data) prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) @@ -193,8 +192,9 @@ class ColModernVBertMultiModalProcessor( result.update(image_outputs) processed_data = result - processed_data.update(passthrough_data) - return processed_data + return self._finalize_hf_mm_data( + mm_data, hf_kwargs, passthrough_data, processed_data + ) def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/colpali.py b/vllm/model_executor/models/colpali.py index aea809c15e..4add3b83a6 100644 --- a/vllm/model_executor/models/colpali.py +++ b/vllm/model_executor/models/colpali.py @@ -20,7 +20,7 @@ from collections.abc import Iterable, Mapping import torch import torch.nn as nn -from transformers import BatchFeature, PaliGemmaProcessor +from transformers import PaliGemmaProcessor from vllm.config import VllmConfig from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed @@ -28,6 +28,7 @@ from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.parse import MultiModalDataItems +from vllm.multimodal.processing.processor import HFMultiModalInputs from .interfaces import SupportsLateInteraction from .interfaces_base import default_pooling_type @@ -59,31 +60,19 @@ class ColPaliProcessingInfo(PaliGemmaProcessingInfo): class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor): """Multimodal processor for ColPali.""" - def _apply_hf_processor_main( + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - if not mm_data: - return BatchFeature(dict(passthrough_data)) + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) # The ColPali tokenizer_config.json ships with a small default # max_length (50) that truncates the 1024 image tokens inserted # by PaliGemmaProcessor, causing a token-count mismatch. # vLLM enforces its own max_model_len, so we disable HF # truncation to keep all image + text tokens intact. - processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - mm_data, - dict(**hf_processor_mm_kwargs, truncation=False), - ) - processed_data.update(passthrough_data) - return processed_data + return hf_inputs._replace(hf_kwargs=dict(hf_inputs.hf_kwargs, truncation=False)) @default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL") diff --git a/vllm/model_executor/models/deepseek_ocr.py b/vllm/model_executor/models/deepseek_ocr.py index 6e43868095..3ead25f173 100644 --- a/vllm/model_executor/models/deepseek_ocr.py +++ b/vllm/model_executor/models/deepseek_ocr.py @@ -46,10 +46,10 @@ from vllm.multimodal.processing import ( PromptReplacement, PromptUpdate, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.sampling_params import SamplingParams from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config -from vllm.tokenizers.hf import HfTokenizer from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config from vllm.transformers_utils.processors.deepseek_ocr import ( BASE_SIZE, @@ -287,34 +287,19 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing class DeepseekOCRMultiModalProcessor( BaseMultiModalProcessor[DeepseekOCRProcessingInfo] ): - def _apply_hf_processor_main( + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) + if "text" in hf_inputs.hf_data: + hf_inputs.hf_data["prompt"] = hf_inputs.hf_data.pop("text") - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - - if mm_data: - processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - dict(prompt=prompt_text, **mm_data), - hf_processor_mm_kwargs, - ) - - else: - tokenizer = self.info.get_tokenizer() - assert isinstance(tokenizer, HfTokenizer) - processed_data = tokenizer( - prompt_text, add_special_tokens=True, return_tensors="pt" - ) - - processed_data.update(passthrough_data) - return processed_data + return hf_inputs def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/deepseek_ocr2.py b/vllm/model_executor/models/deepseek_ocr2.py index caaf909671..a64b71b64e 100644 --- a/vllm/model_executor/models/deepseek_ocr2.py +++ b/vllm/model_executor/models/deepseek_ocr2.py @@ -45,9 +45,9 @@ from vllm.multimodal.processing import ( PromptReplacement, PromptUpdate, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config -from vllm.tokenizers.hf import HfTokenizer from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config from vllm.transformers_utils.processors.deepseek_ocr import ( BASE_SIZE, @@ -158,35 +158,19 @@ class DeepseekOCR2DummyInputsBuilder( class DeepseekOCR2MultiModalProcessor( BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo] ): - def _apply_hf_processor_main( + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) + if "text" in hf_inputs.hf_data: + hf_inputs.hf_data["prompt"] = hf_inputs.hf_data.pop("text") - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - - if mm_data: - processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - dict(prompt=prompt_text, **mm_data), - hf_processor_mm_kwargs, - ) - - else: - tokenizer = self.info.get_tokenizer() - assert isinstance(tokenizer, HfTokenizer) - processed_data = tokenizer( - prompt_text, add_special_tokens=True, return_tensors="pt" - ) - - processed_data.update(passthrough_data) - - return processed_data + return hf_inputs def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/deepseek_vl2.py b/vllm/model_executor/models/deepseek_vl2.py index 5fb52cff90..6f798a87e6 100644 --- a/vllm/model_executor/models/deepseek_vl2.py +++ b/vllm/model_executor/models/deepseek_vl2.py @@ -35,7 +35,7 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, - MultiModalProcessingInfo, + MultiModalProcessingResult, ProcessorInputs, PromptReplacement, PromptUpdate, @@ -233,7 +233,7 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing class DeepseekVL2MultiModalProcessor( BaseMultiModalProcessor[DeepseekVL2ProcessingInfo] ): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _postprocess_hf_mm_data( @@ -307,7 +307,7 @@ class DeepseekVL2MultiModalProcessor( self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> MultiModalProcessingInfo: + ) -> MultiModalProcessingResult: # The processor logic is different for len(images) <= 2 vs > 2 # Since the processing cache assumes that the processor output is # invariant of how many images are passed per prompt, we only diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index 81907a517d..e1b847c294 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -71,6 +71,7 @@ from vllm.multimodal.processing import ( PromptUpdate, cached_encode, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.sequence import IntermediateTensors from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -1112,34 +1113,32 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin pixel_values = pixel_values.to(hf_config.dtype) return pixel_values - def _apply_hf_processor_main( + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) + hf_data = hf_inputs.hf_data - if "images" not in mm_data and "videos" not in mm_data: - return BatchFeature(dict(passthrough_data)) + if "images" not in hf_data and "videos" not in hf_data: + return hf_inputs - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - - mm_data = dict(mm_data) - if "images" not in mm_data: - mm_data["images"] = [] - if "videos" not in mm_data: - mm_data["videos"] = [] + prompt_text = hf_data["text"] + hf_data["text"] = [prompt_text] + hf_data.setdefault("images", []) + hf_data.setdefault("videos", []) # Check if HF processor supports video metadata - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_inputs.hf_kwargs) supports_video_metadata = getattr( hf_processor, "supports_video_metadata", False ) - videos = mm_data["videos"] + videos = hf_data["videos"] assert isinstance(videos, Sequence) if videos and not supports_video_metadata: # Old HF processor, unwrap tuple to pure frames @@ -1147,49 +1146,43 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin "HF processor doesn't support video metadata. " "Timestamps will NOT be rendered. Please upgrade the model." ) - mm_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos] + hf_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos] - processor_output = self.info.ctx.call_hf_processor( - hf_processor, - dict( - text=[prompt_text], images=mm_data["images"], videos=mm_data["videos"] - ), - hf_processor_mm_kwargs, - ) + return hf_inputs + + def _postprocess_hf_mm_data( + self, + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if "images" not in hf_data and "videos" not in hf_data: + return processed_data # Divide the processor_output into two modalities: image and video. - if processor_output is not None: - pixel_values = processor_output["images"] - if pixel_values is not None: - processor_output["images"] = self._pixel_values_norm( - pixel_values, hf_processor_mm_kwargs - ) - for key in list(processor_output.keys()): - if processor_output[key] is None: - del processor_output[key] - continue - if key == "grid_thw": - grid_thw = processor_output["grid_thw"] - pixel_values_all = processor_output["images"] - # Identify elements where the first - # dimension is greater than 1 and - # treat them as the video modality - mask = grid_thw[:, 0] > 1 - processor_output["video_grid_thw"] = grid_thw[mask] - processor_output["image_grid_thw"] = grid_thw[~mask] - image_patch_num = ( - processor_output["image_grid_thw"].prod(dim=1).sum() - ) - processor_output["pixel_values"] = pixel_values_all[ - :image_patch_num - ] - processor_output["pixel_values_videos"] = pixel_values_all[ - image_patch_num: - ] - del processor_output["images"] + pixel_values = processed_data["images"] + if pixel_values is not None: + processed_data["images"] = self._pixel_values_norm(pixel_values, hf_kwargs) + for key in list(processed_data.keys()): + if processed_data[key] is None: + del processed_data[key] + continue + if key == "grid_thw": + grid_thw = processed_data["grid_thw"] + pixel_values_all = processed_data["images"] + # Identify elements where the first + # dimension is greater than 1 and + # treat them as the video modality + mask = grid_thw[:, 0] > 1 + processed_data["video_grid_thw"] = grid_thw[mask] + processed_data["image_grid_thw"] = grid_thw[~mask] + image_patch_num = processed_data["image_grid_thw"].prod(dim=1).sum() + processed_data["pixel_values"] = pixel_values_all[:image_patch_num] + processed_data["pixel_values_videos"] = pixel_values_all[ + image_patch_num: + ] + del processed_data["images"] - processed_data = processor_output - processed_data.update(passthrough_data) return processed_data def _get_prompt_updates( diff --git a/vllm/model_executor/models/fireredasr2.py b/vllm/model_executor/models/fireredasr2.py index 847ec89e8f..82a4a6b506 100644 --- a/vllm/model_executor/models/fireredasr2.py +++ b/vllm/model_executor/models/fireredasr2.py @@ -38,6 +38,7 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.transformers_utils.processor import cached_processor_from_config from vllm.transformers_utils.processors.fireredasr2 import ( FireRedASR2FeatureExtractor, @@ -228,37 +229,24 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing class FireRedASR2MultiModalProcessor( BaseMultiModalProcessor[FireRedASR2ProcessingInfo] ): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _preprocess_hf_mm_data( + def _get_hf_mm_inputs( self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + mm_items: MultiModalDataItems, + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) - mm_data = dict(mm_data) - mm_data["audio"] = mm_data.pop("audios") - - hf_processor_mm_kwargs = dict( - **hf_processor_mm_kwargs, - sampling_rate=feature_extractor.sampling_rate, + feature_extractor = self.info.get_feature_extractor(**hf_kwargs) + return hf_inputs._replace( + hf_kwargs=dict( + hf_inputs.hf_kwargs, + sampling_rate=feature_extractor.sampling_rate, + ) ) - return mm_data, hf_processor_mm_kwargs - - def _postprocess_hf_mm_data( - self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - processed_data: BatchFeature, - ) -> BatchFeature: - if "labels" in processed_data: - processed_data["input_ids"] = processed_data.pop("labels") - - return processed_data - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/funasr.py b/vllm/model_executor/models/funasr.py index 9a7dc49d71..7bf253ab40 100644 --- a/vllm/model_executor/models/funasr.py +++ b/vllm/model_executor/models/funasr.py @@ -49,7 +49,10 @@ from vllm.multimodal.processing import ( PromptUpdate, TimingContext, ) -from vllm.multimodal.processing.processor import MultiModalProcessingInfo +from vllm.multimodal.processing.processor import ( + HFMultiModalInputs, + MultiModalProcessingResult, +) from vllm.transformers_utils.processor import cached_processor_from_config from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -752,42 +755,29 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]): class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _preprocess_hf_mm_data( + def _get_hf_mm_inputs( self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + mm_items: MultiModalDataItems, + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) - mm_data = dict(mm_data) - mm_data["audio"] = mm_data.pop("audios") - - hf_processor_mm_kwargs = dict( - **hf_processor_mm_kwargs, - sampling_rate=feature_extractor.sampling_rate, + feature_extractor = self.info.get_feature_extractor(**hf_kwargs) + return hf_inputs._replace( + hf_kwargs=dict( + hf_inputs.hf_kwargs, + sampling_rate=feature_extractor.sampling_rate, + ) ) - return mm_data, hf_processor_mm_kwargs - - def _postprocess_hf_mm_data( - self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - processed_data: BatchFeature, - ) -> BatchFeature: - if "labels" in processed_data: - processed_data["input_ids"] = processed_data.pop("labels") - - return processed_data - def _cached_apply_hf_processor( self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> MultiModalProcessingInfo: + ) -> MultiModalProcessingResult: # Dithering injects noise into the extracted features, so the # feature extractor is not a pure function of its input. Since the # processing cache assumes that processor outputs are invariant diff --git a/vllm/model_executor/models/funaudiochat.py b/vllm/model_executor/models/funaudiochat.py index 607eb4ea5d..7a093cc2e7 100644 --- a/vllm/model_executor/models/funaudiochat.py +++ b/vllm/model_executor/models/funaudiochat.py @@ -616,29 +616,29 @@ class FunAudioChatMultiModalProcessor( def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) if not mm_data: - return BatchFeature(dict(passthrough_data)) + return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data) prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) tokenizer = self.info.get_tokenizer() input_ids = torch.tensor([tokenizer.encode(prompt_text)]) - audios = mm_data.get("audios", []) + audios = mm_data.get("audio", []) if not audios: processed_data = BatchFeature({"input_ids": input_ids}) - processed_data.update(passthrough_data) - return processed_data + return self._finalize_hf_mm_data( + mm_data, hf_kwargs, passthrough_data, processed_data + ) assert isinstance(audios, Sequence) - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + feature_extractor = self.info.get_feature_extractor(**hf_kwargs) sr = int(feature_extractor.sampling_rate) min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400) @@ -690,8 +690,9 @@ class FunAudioChatMultiModalProcessor( } processed_data = BatchFeature({"input_ids": input_ids, **mm_inputs}) - processed_data.update(passthrough_data) - return processed_data + return self._finalize_hf_mm_data( + mm_data, hf_kwargs, passthrough_data, processed_data + ) def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index d6aaabe023..5bccd9aec0 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -263,7 +263,7 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]): class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _postprocess_hf_mm_data( diff --git a/vllm/model_executor/models/gemma3n_mm.py b/vllm/model_executor/models/gemma3n_mm.py index 769c4c666a..dfde046fec 100644 --- a/vllm/model_executor/models/gemma3n_mm.py +++ b/vllm/model_executor/models/gemma3n_mm.py @@ -265,20 +265,9 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]): class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _preprocess_hf_mm_data( - self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - mm_data = dict(mm_data) - if "audios" in mm_data: - mm_data["audio"] = mm_data.pop("audios") - - return mm_data, hf_processor_mm_kwargs - def _postprocess_hf_mm_data( self, mm_data: Mapping[str, object], diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index 752e2243df..6ad98cb650 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -595,22 +595,25 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - if not mm_data: - return BatchFeature(dict(passthrough_data)) + if not hf_data: + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data) - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + prompt_text = hf_data.pop("text") + assert isinstance(prompt_text, str) - merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs) + merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_kwargs) val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs) if val is not None and val not in _SUPPORTED_SOFT_TOKENS: @@ -619,14 +622,12 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): f"Valid values are {_SUPPORTED_SOFT_TOKENS}." ) - mm_data = dict(mm_data) - # ---- VIDEO HANDLING ---- # Gemma4 decomposes video into timestamped image frames. # Each frame is processed with max_soft_tokens=70 through the # same vision tower, matching transformers processing_gemma4.py. video_outputs: dict[str, Any] = {} - if videos := mm_data.pop("videos", []): + if videos := hf_data.pop("videos", []): assert isinstance(videos, list) processor = self.info.get_hf_processor() @@ -656,7 +657,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): timestamps = [idx / fps for idx in frame_indices] # Process frames as images with max_soft_tokens=70 - video_mm_kwargs = dict(hf_processor_mm_kwargs) + video_mm_kwargs = dict(hf_kwargs) # Override configured image options too: these inputs are video # frames, whose cache keys exclude images_kwargs. video_mm_kwargs["images_kwargs"] = {} @@ -726,18 +727,14 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): "video_timestamps": video_timestamps_per_video, } - # The processor accepts 'audio' not 'audios'. - if "audios" in mm_data: - mm_data["audio"] = mm_data.pop("audios") - # Warn if any audio waveform exceeds the model's max duration. - if "audio" in mm_data: + if "audio" in hf_data: processor = self.info.get_hf_processor() sr = processor.feature_extractor.sampling_rate max_tokens = processor.audio_seq_length ms_per_tok = processor.audio_ms_per_token max_duration_s = max_tokens * ms_per_tok / 1000.0 - audios = mm_data["audio"] + audios = hf_data["audio"] if not isinstance(audios, (list, tuple)): audios = [audios] for i, waveform in enumerate(audios): @@ -760,13 +757,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): # NOTE: This requires a corresponding type annotation on the # HF side (Gemma4ProcessorKwargs.images_kwargs) so that # _merge_kwargs routes max_soft_tokens into images_kwargs. - patched_mm_kwargs = dict(hf_processor_mm_kwargs) + patched_mm_kwargs = dict(hf_kwargs) if val is not None and is_top_level_max_soft_tokens: patched_mm_kwargs["max_soft_tokens"] = val processed_data = self.info.ctx.call_hf_processor( self.info.get_hf_processor(**patched_mm_kwargs), - dict(text=prompt_text, **mm_data), + dict(text=prompt_text, **hf_data), patched_mm_kwargs, ) @@ -797,9 +794,9 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): processed_data["input_features_mask"] = unpadded_masks processed_data.update(video_outputs) - processed_data.update(passthrough_data) - - return processed_data + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data + ) def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 87baaa1069..c0f77e770d 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -1582,52 +1582,59 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): return prepared_data, prepared_kwargs + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - if not mm_data: - return BatchFeature(dict(passthrough_data)) + if not hf_data: + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data) - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + prompt_text = hf_data.pop("text") + assert isinstance(prompt_text, str) - mm_data = dict(mm_data) - - processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + processor = self.info.get_hf_processor(**hf_kwargs) use_direct_path = ( not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA ) if use_direct_path: prepared_data, prepared_kwargs = self._get_direct_path_inputs( - mm_data, hf_processor_mm_kwargs + hf_data, hf_kwargs ) processed_data = self.info.ctx.call_hf_processor( self.info.get_hf_processor(**prepared_kwargs), dict(text=prompt_text, **prepared_data), prepared_kwargs, ) - processed_data.update(passthrough_data) - return processed_data + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data + ) - videos = mm_data.get("videos") - if isinstance(videos, list) and videos: + if ( + "videos" in hf_data + and isinstance(videos := hf_data["videos"], list) + and len(videos) > 0 + ): + # Videos are processed item-wise below; remove them from hf_data so + # the combined call at the end does not re-sample the raw frames. + del hf_data["videos"] video_grid_thw_lst = [] pixel_values_videos_lst = [] frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor) swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id - mm_data.pop("videos") for item in videos: video_array, metadata = item # don't update mm_kwargs inplace - video_mm_kwargs = dict(**hf_processor_mm_kwargs) + video_mm_kwargs = dict(**hf_kwargs) video_mm_kwargs["do_sample_frames"] = metadata.get( "do_sample_frames", True ) @@ -1667,9 +1674,9 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): swap_video_frame_tokens = False processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - dict(text=prompt_text, **mm_data), - hf_processor_mm_kwargs, + self.info.get_hf_processor(**hf_kwargs), + dict(text=prompt_text, **hf_data), + hf_kwargs, ) if swap_video_frame_tokens: input_ids = processed_data["input_ids"] @@ -1677,9 +1684,9 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): processed_data["input_ids"] = input_ids processed_data.update(video_outputs) - processed_data.update(passthrough_data) - - return processed_data + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data + ) def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/glmasr.py b/vllm/model_executor/models/glmasr.py index 710c6378a8..003b02c28e 100644 --- a/vllm/model_executor/models/glmasr.py +++ b/vllm/model_executor/models/glmasr.py @@ -44,6 +44,7 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config from vllm.transformers_utils.processor import cached_processor_from_config @@ -739,26 +740,24 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"]) chunk_counts.append(min(n_chunks, max_windows)) return chunk_counts - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _preprocess_hf_mm_data( + def _get_hf_mm_inputs( self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - mm_data = dict(mm_data) - if "audios" in mm_data: - mm_data["audio"] = mm_data.pop("audios") + mm_items: MultiModalDataItems, + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) - hf_processor_mm_kwargs = dict( - **hf_processor_mm_kwargs, - sampling_rate=feature_extractor.sampling_rate, + feature_extractor = self.info.get_feature_extractor(**hf_kwargs) + return hf_inputs._replace( + hf_kwargs=dict( + hf_inputs.hf_kwargs, + sampling_rate=feature_extractor.sampling_rate, + ) ) - return mm_data, hf_processor_mm_kwargs - def _postprocess_hf_mm_data( self, mm_data: Mapping[str, object], diff --git a/vllm/model_executor/models/granite_speech.py b/vllm/model_executor/models/granite_speech.py index c8d1f7111b..529befd21d 100644 --- a/vllm/model_executor/models/granite_speech.py +++ b/vllm/model_executor/models/granite_speech.py @@ -110,7 +110,7 @@ class GraniteSpeechAudioInputs(TensorSchema): """List of audio embedding sizes for each item in batch.""" -class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo): +class GraniteSpeechProcessingInfo(BaseProcessingInfo): def get_data_parser(self): feature_extractor = self.get_hf_processor().audio_processor @@ -135,7 +135,7 @@ class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo): ### Input Processing & Multimodal utils class GraniteSpeechMultiModalProcessor( - BaseMultiModalProcessor[GraniteSpeechMultiModalProcessingInfo] + BaseMultiModalProcessor[GraniteSpeechProcessingInfo] ): def _get_mm_fields_config( self, @@ -181,23 +181,9 @@ class GraniteSpeechMultiModalProcessor( ) ] - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _preprocess_hf_mm_data( - self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - mm_data = dict(mm_data) - audios = mm_data.pop("audios", []) - - if audios: - # GraniteSpeechFeatureExtractor accepts "audio" - mm_data["audio"] = audios - - return mm_data, hf_processor_mm_kwargs - def _postprocess_hf_mm_data( self, mm_data: Mapping[str, object], @@ -216,7 +202,7 @@ class GraniteSpeechMultiModalProcessor( class GraniteSpeechDummyInputsBuilder( - BaseDummyInputsBuilder[GraniteSpeechMultiModalProcessingInfo] + BaseDummyInputsBuilder[GraniteSpeechProcessingInfo] ): def get_dummy_mm_data( self, @@ -565,7 +551,7 @@ class GraniteSpeechCTCEncoder(nn.Module): @MULTIMODAL_REGISTRY.register_processor( GraniteSpeechMultiModalProcessor, - info=GraniteSpeechMultiModalProcessingInfo, + info=GraniteSpeechProcessingInfo, dummy_inputs=GraniteSpeechDummyInputsBuilder, ) class GraniteSpeechForConditionalGeneration( diff --git a/vllm/model_executor/models/granite_speech_plus.py b/vllm/model_executor/models/granite_speech_plus.py index ba95cfed13..7e695b7ed5 100644 --- a/vllm/model_executor/models/granite_speech_plus.py +++ b/vllm/model_executor/models/granite_speech_plus.py @@ -12,8 +12,8 @@ from .granite_speech import ( GraniteSpeechCTCEncoder, GraniteSpeechDummyInputsBuilder, GraniteSpeechForConditionalGeneration, - GraniteSpeechMultiModalProcessingInfo, GraniteSpeechMultiModalProcessor, + GraniteSpeechProcessingInfo, ) ISO639_1_SUPPORTED_LANGS = { @@ -60,7 +60,7 @@ class GraniteSpeechPlusCTCEncoder(GraniteSpeechCTCEncoder): @MULTIMODAL_REGISTRY.register_processor( GraniteSpeechMultiModalProcessor, - info=GraniteSpeechMultiModalProcessingInfo, + info=GraniteSpeechProcessingInfo, dummy_inputs=GraniteSpeechDummyInputsBuilder, ) class GraniteSpeechPlusForConditionalGeneration(GraniteSpeechForConditionalGeneration): diff --git a/vllm/model_executor/models/h2ovl.py b/vllm/model_executor/models/h2ovl.py index 5be2d7b46d..596f24a3eb 100644 --- a/vllm/model_executor/models/h2ovl.py +++ b/vllm/model_executor/models/h2ovl.py @@ -21,7 +21,7 @@ from vllm.multimodal.parse import ( MultiModalDataItems, ) from vllm.multimodal.processing.processor import ( - MultiModalProcessingInfo, + MultiModalProcessingResult, ProcessorInputs, PromptReplacement, TimingContext, @@ -139,7 +139,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> MultiModalProcessingInfo: + ) -> MultiModalProcessingResult: # The processor logic is different for len(images) <= 1 vs > 1 # Since the processing cache assumes that the processor output is # invariant of how many images are passed per prompt, we only diff --git a/vllm/model_executor/models/hyperclovax_vision_v2.py b/vllm/model_executor/models/hyperclovax_vision_v2.py index 10c066cb71..07106c9471 100644 --- a/vllm/model_executor/models/hyperclovax_vision_v2.py +++ b/vllm/model_executor/models/hyperclovax_vision_v2.py @@ -261,44 +261,8 @@ class HCXVisionV2MultiModalProcessor( ): """Multimodal processor for HyperCLOVAX V2 (32B Think model).""" - def _apply_hf_processor_main( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - if not mm_data: - return BatchFeature(dict(passthrough_data)) - - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - - images = mm_data.get("images") - videos = mm_data.get("videos") - - # Get the HF processor - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - - # Build data dict for HF processor (images/videos only) - # NOTE: We pass the prompt as-is without token normalization. - # Token expansion is handled by vLLM via _get_prompt_updates. - data: dict[str, object] = dict( - text=prompt_text, - images=images, - videos=videos, - ) - - processed_data = self.info.ctx.call_hf_processor( - hf_processor=hf_processor, - data=data, - kwargs=hf_processor_mm_kwargs, - ) - processed_data.update(passthrough_data) - - return processed_data + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index 014c4ae6b5..dd03e17d1e 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -52,6 +52,7 @@ from vllm.multimodal.processing import ( PromptUpdateDetails, cached_encode, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.sequence import IntermediateTensors from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -240,31 +241,32 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]) class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]): - def _apply_hf_processor_main( + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - if not mm_data: - return BatchFeature(dict(passthrough_data)) + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) image_processor = self.info.get_hf_processor().image_processor if getattr(image_processor, "backend", "pil") == "pil": - hf_processor_mm_kwargs = { - "input_data_format": "channels_last", - **hf_processor_mm_kwargs, - } + hf_inputs = hf_inputs._replace( + hf_kwargs={ + "input_data_format": "channels_last", + **hf_inputs.hf_kwargs, + } + ) - processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - mm_data, - hf_processor_mm_kwargs, - ) + return hf_inputs + + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if not mm_data: + return processed_data images = mm_data.get("images", []) mm_items = self.info.parse_mm_data({"image": images}, validate=False) @@ -289,8 +291,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo processed_data["pixel_values"].squeeze_(0) processed_data["pixel_attention_mask"].squeeze_(0) - processed_data.update(passthrough_data) - return processed_data def _get_mm_fields_config( diff --git a/vllm/model_executor/models/interns1.py b/vllm/model_executor/models/interns1.py index 057a6ce247..4a4264660d 100644 --- a/vllm/model_executor/models/interns1.py +++ b/vllm/model_executor/models/interns1.py @@ -335,28 +335,30 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo]) class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]): """Basic image-only MultiModalProcessor for InternS1-style models.""" + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - if not mm_data: - return BatchFeature(dict(passthrough_data)) + if not hf_data: + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data) - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + prompt_text = hf_data.pop("text") + assert isinstance(prompt_text, str) - mm_data = dict(mm_data) - videos = mm_data.pop("videos", []) - images = mm_data.pop("images", []) + videos = hf_data.pop("videos", []) + images = hf_data.pop("images", []) assert isinstance(videos, list) assert isinstance(images, list) - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_kwargs) tokenizer = hf_processor.tokenizer vocab = tokenizer.get_vocab() @@ -374,9 +376,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo image_pixel_values = [] for image in images: processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), + self.info.get_hf_processor(**hf_kwargs), dict(text=hf_processor.image_token, **{"images": image}), - hf_processor_mm_kwargs, + hf_kwargs, ) image_pixel_values.append(processed_data.pop("pixel_values")) @@ -398,9 +400,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo video_pixel_values = [] for video in videos: processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), + self.info.get_hf_processor(**hf_kwargs), dict(text=hf_processor.video_token, **{"videos": video}), - hf_processor_mm_kwargs, + hf_kwargs, ) video_pixel_values.append(processed_data.pop("pixel_values")) @@ -430,9 +432,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo processed_data.update(text_outputs) processed_data.update(image_outputs) processed_data.update(video_outputs) - processed_data.update(passthrough_data) - - return processed_data + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data + ) def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/jina_vl.py b/vllm/model_executor/models/jina_vl.py index 2895ec13b9..9712845619 100644 --- a/vllm/model_executor/models/jina_vl.py +++ b/vllm/model_executor/models/jina_vl.py @@ -13,7 +13,7 @@ from vllm.model_executor.layers.pooler import DispatchPooler from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalKwargsItems from vllm.multimodal.processing.processor import ( - MultiModalProcessingInfo, + MultiModalProcessingResult, ProcessorInputs, TimingContext, ) @@ -63,19 +63,19 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor): self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> MultiModalProcessingInfo: - mm_info = super()._cached_apply_hf_processor(inputs, timing_ctx) + ) -> MultiModalProcessingResult: + mm_res = super()._cached_apply_hf_processor(inputs, timing_ctx) # Score inputs are query-first, while the prompt template is document-first. mm_kwargs = MultiModalKwargsItems( { modality: list(reversed(items)) - for modality, items in mm_info.kwargs.items() + for modality, items in mm_res.kwargs.items() } ) mm_hashes = { modality: list(reversed(hashes)) - for modality, hashes in mm_info.hashes.items() + for modality, hashes in mm_res.hashes.items() } mm_prompt_updates = { modality: [ @@ -85,10 +85,11 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor): ] for item_idx, updates in enumerate(reversed(item_updates)) ] - for modality, item_updates in mm_info.prompt_updates.items() + for modality, item_updates in mm_res.prompt_updates.items() } - return MultiModalProcessingInfo( + return MultiModalProcessingResult( + prompt_ids=mm_res.prompt_ids, kwargs=mm_kwargs, hashes=mm_hashes, prompt_updates=mm_prompt_updates, diff --git a/vllm/model_executor/models/kanana_v.py b/vllm/model_executor/models/kanana_v.py index a9a5dfda24..e3c27e479b 100644 --- a/vllm/model_executor/models/kanana_v.py +++ b/vllm/model_executor/models/kanana_v.py @@ -459,24 +459,27 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo]) def media_token_id(self) -> int: return self.info.get_hf_config().text_config.eos_token_id + 1 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: """Run the underlying HF processor on text and image data.""" - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - if not mm_data or not mm_data.get("images", []): - return BatchFeature(dict(passthrough_data)) + if not hf_data or not hf_data.get("images", []): + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data) - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + prompt_text = hf_data.pop("text") + assert isinstance(prompt_text, str) # Images - image_inputs = mm_data.get("images", []) + image_inputs = hf_data.get("images", []) pixel_sizes = [] if not isinstance(image_inputs[0], Image.Image): image_inputs = [Image.fromarray(image) for image in image_inputs] @@ -530,8 +533,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo]) pixel_sizes=torch.tensor(pixel_sizes), ) processed_data = BatchFeature(combined_outputs, tensor_type="pt") - processed_data.update(passthrough_data) - return processed_data + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data + ) def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index 6bbffae029..b11adfa499 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -1160,7 +1160,7 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]): class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _get_prompt_updates( diff --git a/vllm/model_executor/models/keye_vl1_5.py b/vllm/model_executor/models/keye_vl1_5.py index eec0630455..b52d50296a 100644 --- a/vllm/model_executor/models/keye_vl1_5.py +++ b/vllm/model_executor/models/keye_vl1_5.py @@ -373,7 +373,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo): class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _get_prompt_updates( diff --git a/vllm/model_executor/models/kimi_audio.py b/vllm/model_executor/models/kimi_audio.py index 71cf389f98..219c7f0e7b 100644 --- a/vllm/model_executor/models/kimi_audio.py +++ b/vllm/model_executor/models/kimi_audio.py @@ -47,6 +47,7 @@ from vllm.multimodal.processing import ( ) from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, + HFMultiModalInputs, ProcessorInputs, ) from vllm.sequence import IntermediateTensors @@ -227,27 +228,17 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser): class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]): """vLLM multi-modal processor wrapper for Kimi-Audio.""" - def _apply_hf_processor_main( + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - """Call the HuggingFace processor.""" - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - if not mm_data: - return BatchFeature(dict(passthrough_data)) - - # Convert mm_data format: {'audios': [...]} -> {'audio': ...} - mm_data = dict(mm_data) - audios = mm_data.pop("audios", []) + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) # Convert audio format: [(array, sr), ...] -> [array, ...] # KimiAudioProcessor expects raw numpy arrays - if audios: + mm_data = hf_inputs.hf_data + if audios := mm_data.pop("audio", []): audio_arrays = [] for aud in audios: if isinstance(aud, (tuple, list)) and len(aud) == 2: @@ -259,14 +250,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn audio_arrays.append(aud) mm_data["audio"] = audio_arrays - # Use the context's call_hf_processor for proper handling - processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - mm_data, - hf_processor_mm_kwargs, - ) - processed_data.update(passthrough_data) - return processed_data + return hf_inputs def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/kimi_vl.py b/vllm/model_executor/models/kimi_vl.py index 9aba4d1945..344dfdaaf0 100644 --- a/vllm/model_executor/models/kimi_vl.py +++ b/vllm/model_executor/models/kimi_vl.py @@ -236,7 +236,7 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]): class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _get_mm_fields_config( diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index 9b24095ab6..1cd51b93a2 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -406,7 +406,7 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]): class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _postprocess_hf_mm_data( diff --git a/vllm/model_executor/models/lightonocr.py b/vllm/model_executor/models/lightonocr.py index 0ce0d7f3d2..d8a1240242 100644 --- a/vllm/model_executor/models/lightonocr.py +++ b/vllm/model_executor/models/lightonocr.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Iterable, Mapping, Sequence -from typing import TypeVar import torch import torch.nn as nn @@ -35,8 +34,6 @@ from vllm.multimodal.processing import ( PromptUpdateDetails, ) -_I = TypeVar("_I", bound=Mistral3ProcessingInfo) - class LightOnOCRProcessingInfo(Mistral3ProcessingInfo): def get_vision_encoder_info( @@ -49,8 +46,8 @@ class LightOnOCRProcessingInfo(Mistral3ProcessingInfo): return PixtralHFEncoderInfo(self.get_hf_config()) -class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: +class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessingInfo]): + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _postprocess_hf_mm_data( @@ -62,27 +59,6 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn if not mm_data: return processed_data - input_ids = processed_data.get("input_ids") - if input_ids is not None: - processor = self.info.get_hf_processor() - tokenizer = self.info.get_tokenizer() - vocab = tokenizer.get_vocab() - - break_id = vocab.get(processor.image_break_token) - end_id = vocab.get(processor.image_end_token) - - # create mask to remove break/end tokens - keep_mask = ~torch.isin( - input_ids, - torch.tensor([break_id, end_id]), - ) - - processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0) - if "attention_mask" in processed_data: - processed_data["attention_mask"] = processed_data["attention_mask"][ - keep_mask - ].unsqueeze(0) - # un-pad pixel_values per-image so caches remain independent. pixel_values = processed_data.get("pixel_values") if pixel_values is not None: diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index 1c4ebff7a0..c3a437eb36 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -320,7 +320,7 @@ class PixtralHFProcessingInfo(BaseLlavaProcessingInfo): class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _postprocess_hf_mm_data( diff --git a/vllm/model_executor/models/llava_next.py b/vllm/model_executor/models/llava_next.py index b7b1a8533d..02400a763b 100644 --- a/vllm/model_executor/models/llava_next.py +++ b/vllm/model_executor/models/llava_next.py @@ -209,7 +209,7 @@ class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]): class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _get_mm_fields_config( diff --git a/vllm/model_executor/models/llava_onevision.py b/vllm/model_executor/models/llava_onevision.py index 55f63c49ba..66503d7741 100644 --- a/vllm/model_executor/models/llava_onevision.py +++ b/vllm/model_executor/models/llava_onevision.py @@ -324,33 +324,36 @@ class LlavaOnevisionMultiModalProcessor( pixel_values_videos=MultiModalFieldConfig.batched("video"), ) + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - if not mm_data: - return BatchFeature(dict(passthrough_data)) + if not hf_data: + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data) - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + prompt_text = hf_data.pop("text") + assert isinstance(prompt_text, str) - mm_data = dict(mm_data) - videos = mm_data.pop("videos", []) + videos = hf_data.pop("videos", []) assert isinstance(videos, list) if not videos: processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - dict(text=prompt_text, **mm_data), - hf_processor_mm_kwargs, + self.info.get_hf_processor(**hf_kwargs), + dict(text=prompt_text, **hf_data), + hf_kwargs, + ) + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data ) - processed_data.update(passthrough_data) - return processed_data # LLaVA-OneVision processor doesn't support multiple videos # with different sizes when converting back to tensors @@ -360,13 +363,13 @@ class LlavaOnevisionMultiModalProcessor( image_token = processor.image_token video_token = processor.video_token - images = mm_data.pop("images", []) + images = hf_data.pop("images", []) assert isinstance(images, list) if images: processor_outputs = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), + self.info.get_hf_processor(**hf_kwargs), dict(text=image_token * len(images), **{"images": images}), - hf_processor_mm_kwargs, + hf_kwargs, ) image_outputs = { k: v @@ -379,9 +382,9 @@ class LlavaOnevisionMultiModalProcessor( pixel_values_videos = [] for video in videos: item_outputs = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), + self.info.get_hf_processor(**hf_kwargs), dict(text=video_token, **{"videos": video}), - hf_processor_mm_kwargs, + hf_kwargs, ) pixel_values_videos.append(item_outputs["pixel_values_videos"][0]) @@ -394,8 +397,9 @@ class LlavaOnevisionMultiModalProcessor( **video_outputs, ) processed_data = BatchFeature(combined_outputs) - processed_data.update(passthrough_data) - return processed_data + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data + ) def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/llava_onevision2.py b/vllm/model_executor/models/llava_onevision2.py index 71c3bc79dc..6988982c23 100644 --- a/vllm/model_executor/models/llava_onevision2.py +++ b/vllm/model_executor/models/llava_onevision2.py @@ -1556,23 +1556,26 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser): class LlavaOnevision2MultiModalProcessor( BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo] ): + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - if not mm_data: - return BatchFeature(dict(passthrough_data)) + if not hf_data: + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data) - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + prompt_text = hf_data.pop("text") + assert isinstance(prompt_text, str) - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_kwargs) + merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_kwargs) merged_kwargs.setdefault("return_tensors", "pt") call_kwargs = { k: v @@ -1589,17 +1592,17 @@ class LlavaOnevision2MultiModalProcessor( "codec_config", } } - mm_data = dict(mm_data) + # Explicit None + length checks: ``mm_data[...]`` may be a list, numpy # array, or tensor, and ``and `` would raise on the ambiguous # truth value of a multi-element array. - _videos = mm_data.get("videos") + _videos = hf_data.get("videos") if _videos is not None and not isinstance(_videos, Sized): raise TypeError(f"object of type '{type(_videos).__name__}' has no len()") videos_present = _videos is not None and len(_videos) > 0 codec_video_paths = ( - _extract_codec_video_paths(mm_data["videos"]) if videos_present else None + _extract_codec_video_paths(hf_data["videos"]) if videos_present else None ) is_codec_marker = codec_video_paths is not None # Fallback: caller passed video_backend=codec via mm_processor_kwargs @@ -1613,7 +1616,7 @@ class LlavaOnevision2MultiModalProcessor( and call_kwargs.get("video_backend") == "codec" ) if is_codec_kwarg: - raw = mm_data["videos"] + raw = hf_data["videos"] if isinstance(raw, str): codec_video_paths = [raw] elif isinstance(raw, (list, tuple)) and all( @@ -1638,7 +1641,7 @@ class LlavaOnevision2MultiModalProcessor( # Codec backend: HF processor consumes the path string directly and # performs decode + canvas-packing internally. The dummy ndarray # we attached during prepare_codec_video_input is discarded here. - mm_data["videos"] = ( + hf_data["videos"] = ( codec_video_paths if len(codec_video_paths) > 1 else codec_video_paths[0] @@ -1648,11 +1651,9 @@ class LlavaOnevision2MultiModalProcessor( # ``video_backend="codec"`` via mm_kwargs so the wrapped processor # dispatches to its codec branch. output = self.info.ctx.call_hf_processor( - self.info.get_hf_processor( - **{**hf_processor_mm_kwargs, "video_backend": "codec"} - ), - dict(text=prompt_text, **mm_data), - {**hf_processor_mm_kwargs, "video_backend": "codec"}, + self.info.get_hf_processor(**{**hf_kwargs, "video_backend": "codec"}), + dict(text=prompt_text, **hf_data), + {**hf_kwargs, "video_backend": "codec"}, ) data = dict(output) processed_data = BatchFeature( @@ -1660,8 +1661,9 @@ class LlavaOnevision2MultiModalProcessor( data, codec_video_paths, hf_processor ) ) - processed_data.update(passthrough_data) - return processed_data + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data + ) # ---- Frame backend (registered LlavaOnevision2VideoBackend) ------ # Every non-codec video reaches here as a ``(frames_ndarray, metadata)`` @@ -1676,7 +1678,7 @@ class LlavaOnevision2MultiModalProcessor( # provided by the backend's ``compute_frames_index_to_sample``; SSRF / # local-file gating is enforced by the connector before decoding. if videos_present: - timestamp_decimals = hf_processor_mm_kwargs.get( + timestamp_decimals = hf_kwargs.get( "timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS ) if not isinstance( @@ -1690,7 +1692,7 @@ class LlavaOnevision2MultiModalProcessor( per_video_frames: list[list[Image.Image]] = [] per_video_timestamps: list[list[float]] = [] - videos = mm_data["videos"] + videos = hf_data["videos"] if not isinstance(videos, Iterable): raise TypeError(f"Unsupported videos input: {type(videos)}") for item in videos: @@ -1714,7 +1716,7 @@ class LlavaOnevision2MultiModalProcessor( # order) -- otherwise mixed image+video requests bind frames to the # wrong placeholder. ``row_is_video`` labels each grid row so outputs # can be split back into per-modality keys below. - merged_mm_data = dict(mm_data) + merged_mm_data = dict(hf_data) existing_images = merged_mm_data.pop("images", None) caller_images: list[Image.Image] = [] if existing_images: @@ -1760,9 +1762,9 @@ class LlavaOnevision2MultiModalProcessor( # **kwargs to the image processor, so passing the full merged kwarg # set here is a no-op beyond return_tensors/padding. output = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), + self.info.get_hf_processor(**hf_kwargs), dict(text=new_prompt, **merged_mm_data), - hf_processor_mm_kwargs, + hf_kwargs, ) data = dict(output) @@ -1820,20 +1822,22 @@ class LlavaOnevision2MultiModalProcessor( ) processed_data = BatchFeature(data) - processed_data.update(passthrough_data) - return processed_data + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data + ) # ---- Image-only / text-only call -------------------------------- # No videos present: delegate to ``ctx.call_hf_processor``, which # runs the wrapped processor over the (possibly empty) image set and # applies float-tensor dtype postprocessing automatically. processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - dict(text=prompt_text, **mm_data), - hf_processor_mm_kwargs, + self.info.get_hf_processor(**hf_kwargs), + dict(text=prompt_text, **hf_data), + hf_kwargs, + ) + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data ) - processed_data.update(passthrough_data) - return processed_data def _rename_codec_outputs_to_video( self, diff --git a/vllm/model_executor/models/midashenglm.py b/vllm/model_executor/models/midashenglm.py index 300f274a18..7de6ab0e7e 100644 --- a/vllm/model_executor/models/midashenglm.py +++ b/vllm/model_executor/models/midashenglm.py @@ -61,6 +61,7 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.midashenglm import DashengConfig from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -582,35 +583,33 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing class MiDashengLMMultiModalProcessor( BaseMultiModalProcessor[MiDashengLMProcessingInfo] ): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _preprocess_hf_mm_data( + def _get_hf_mm_inputs( self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - mm_data = dict(mm_data) - audios = mm_data.pop("audios", []) - assert isinstance(audios, list) + mm_items: MultiModalDataItems, + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) - min_audio_len = self.info.get_min_audio_len() - processed_audios = [ - np.pad( - audio, - (0, min_audio_len - audio.shape[-1]), - mode="constant", - constant_values=0, - ) - if isinstance(audio, np.ndarray) and audio.shape[-1] < min_audio_len - else audio - for audio in audios - ] + if audios := hf_inputs.hf_data.get("audio"): + assert isinstance(audios, list) - if processed_audios: - mm_data["audio"] = processed_audios + min_audio_len = self.info.get_min_audio_len() + hf_inputs.hf_data["audio"] = [ + np.pad( + audio, + (0, min_audio_len - audio.shape[-1]), + mode="constant", + constant_values=0, + ) + if isinstance(audio, np.ndarray) and audio.shape[-1] < min_audio_len + else audio + for audio in audios + ] - return mm_data, hf_processor_mm_kwargs + return hf_inputs def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/mimo_v2_omni.py b/vllm/model_executor/models/mimo_v2_omni.py index 0edd0736e1..d74edd6d4a 100644 --- a/vllm/model_executor/models/mimo_v2_omni.py +++ b/vllm/model_executor/models/mimo_v2_omni.py @@ -46,6 +46,7 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.transformers_utils.configs.mimo_v2_omni import Mimo_VLVisionConfig from vllm.transformers_utils.processors.mimo_v2_omni import ( MiMoOmniProcessor, @@ -930,27 +931,14 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio") return fields - def _apply_hf_processor_main( + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - """Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo. - Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses - the singular form. - """ - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - if not mm_data: - return BatchFeature(dict(passthrough_data)) - - # Remap audios → audio (MiMoOmniProcessor uses singular param name) - if "audios" in mm_data: - mm_data = {**mm_data, "audio": mm_data["audios"]} - mm_data = {k: v for k, v in mm_data.items() if k != "audios"} + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + """Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.""" + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) + mm_data = hf_inputs.hf_data # Handle video_audio items: convert video part to (TCHW, timestamps) tuple if "video_audio" in mm_data: @@ -992,7 +980,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing audio=va_item.audio, ) ) - mm_data = {**mm_data, "video_audio": va_converted} + mm_data["video_audio"] = va_converted if "videos" in mm_data: converted: list[tuple[torch.Tensor, torch.Tensor]] = [] @@ -1026,15 +1014,9 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing timestamps = torch.arange(T, dtype=torch.float32) / self._INPUT_FPS converted.append((frames, timestamps)) - mm_data = {**mm_data, "videos": converted} + mm_data["videos"] = converted - processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - mm_data, - hf_processor_mm_kwargs, - ) - processed_data.update(passthrough_data) - return processed_data + return hf_inputs def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/minicpmo.py b/vllm/model_executor/models/minicpmo.py index 4251514326..2ec8567921 100644 --- a/vllm/model_executor/models/minicpmo.py +++ b/vllm/model_executor/models/minicpmo.py @@ -445,7 +445,7 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], ) -> Mapping[str, NestedTensors]: - if (audios := mm_data.get("audios")) is None: + if (audios := mm_data.get("audio")) is None: return {} mm_items = self.info.parse_mm_data({"audio": audios}, validate=False) diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index cf818bd77e..14720c5c86 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -1059,19 +1059,18 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) tokenizer = self.info.get_tokenizer() input_ids = torch.tensor([tokenizer.encode(prompt_text)]) - mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs) + mm_inputs = self.process_mm_inputs(mm_data, hf_kwargs) processed_data = BatchFeature( { @@ -1079,8 +1078,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): **mm_inputs, } ) - processed_data.update(passthrough_data) - return processed_data + return self._finalize_hf_mm_data( + mm_data, hf_kwargs, passthrough_data, processed_data + ) def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index 0258fd7bc3..f9d7dafe70 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -248,7 +248,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo): return ImageSize(width=width, height=height) -class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo]): +class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder): def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: num_images = mm_counts.get("image", 0) @@ -280,7 +280,7 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo]) class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _postprocess_hf_mm_data( diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index 027a8db3b5..5c5c272871 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -589,7 +589,7 @@ class Mllama4ProcessingInfo(BaseProcessingInfo): class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _postprocess_hf_mm_data( diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index 8547b689a0..9aeef3715c 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -1122,6 +1122,9 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]): class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]): + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _postprocess_prompt(self, prompt: list[int]) -> list[int]: processor = self.info.get_hf_processor() @@ -1140,28 +1143,28 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]): dict(tokens=tokens), )["input_ids"].tolist() - def _apply_hf_processor_main( + def _call_hf_processor( self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - mm_counts = mm_items.get_all_counts() - - valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0}) - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - if not processor_data: - return BatchFeature(dict(passthrough_data)) - - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - prompt_text = self.dummy_inputs.get_dummy_text(mm_counts) - - processed_data = self.info.ctx.call_hf_processor( + hf_processor = self.info.get_hf_processor(**hf_kwargs) + return self.info.ctx.call_hf_processor( hf_processor.process, - dict(text=prompt_text, **processor_data), - hf_processor_mm_kwargs, + hf_data, + hf_kwargs, ) + def _postprocess_hf_mm_data( + self, + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if not hf_data: + return processed_data + + hf_processor = self.info.get_hf_processor(**hf_kwargs) tokenizer = hf_processor.tokenizer image_patch_id = tokenizer.vocab[IMAGE_PATCH_TOKEN] @@ -1169,7 +1172,7 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]): processed_data.pop("input_ids") - if (images := processor_data.get("images")) is not None: + if (images := hf_data.get("images")) is not None: mm_items = self.info.parse_mm_data({"image": images}, validate=False) parsed_images = mm_items.get_items("image", ImageProcessorItems) image_sizes = [ @@ -1193,8 +1196,6 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]): processed_data["num_crops"] = num_crops processed_data["img_patch_id"] = image_patch_id - processed_data.update(passthrough_data) - return processed_data def _get_mm_fields_config( diff --git a/vllm/model_executor/models/molmo2.py b/vllm/model_executor/models/molmo2.py index 61a9d2b153..4c0ce0868e 100644 --- a/vllm/model_executor/models/molmo2.py +++ b/vllm/model_executor/models/molmo2.py @@ -1932,6 +1932,9 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]): class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _postprocess_prompt(self, prompt: list[int]) -> list[int]: processor = self.info.get_hf_processor() tokenizer = processor.tokenizer @@ -1946,19 +1949,20 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - mm_counts = mm_items.get_all_counts() + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs + ) - valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0}) - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + if not hf_data: + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data) - prompt_text = self.dummy_inputs.get_dummy_text(mm_counts) - - mm_data = dict(processor_data) + prompt_text = hf_data.pop("text") + assert isinstance(prompt_text, str) hf_config = self.info.get_hf_config() - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_kwargs) def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature: res = hf_processor(text=text, images=images, videos=videos, **kwargs) @@ -1973,7 +1977,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): tokenizer = hf_processor.tokenizer image_processor = hf_processor.image_processor - if videos := mm_data.pop("videos", []): + if videos := hf_data.pop("videos", []): assert isinstance(videos, Sequence) bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id @@ -1993,16 +1997,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): # NOTE: metadata.frames_indices indicates # the sampled frames indices of pre-sampled videos, which is # used to calculate the timestamps. Make sure that - # do_sample_frames in hf_processor_mm_kwargs is false for + # do_sample_frames in hf_kwargs is false for # presampled videos. - # NOTE: a copy of hf_processor_mm_kwargs is created to update + # NOTE: a copy of hf_kwargs is created to update # do_sample_frames, otherwise mm_hash for the object will be # incorrect. - video_mm_kwargs = dict(**hf_processor_mm_kwargs) + video_mm_kwargs = dict(**hf_kwargs) if "do_sample_frames" not in video_mm_kwargs: # molmo_utils already has "do_sample_frames" in - # hf_processor_mm_kwargs, don't overwrite it. + # hf_kwargs, don't overwrite it. video_mm_kwargs["do_sample_frames"] = metadata.get( "do_sample_frames", False ) @@ -2067,11 +2071,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): processed_data = self.info.ctx.call_hf_processor( patched_call, - dict(text=prompt_text, **mm_data), - hf_processor_mm_kwargs, + dict(text=prompt_text, **hf_data), + hf_kwargs, ) - if (images := mm_data.get("images")) is not None: + if (images := hf_data.get("images")) is not None: mm_items = self.info.parse_mm_data({"image": images}, validate=False) parsed_images = mm_items.get_items("image", ImageProcessorItems) image_sizes = [ @@ -2113,10 +2117,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): ) processed_data.update(all_video_outputs) - processed_data.update(passthrough_data) processed_data.pop("input_ids") - return processed_data + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data + ) def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/moss_audio.py b/vllm/model_executor/models/moss_audio.py index 64a31375c8..6092812585 100644 --- a/vllm/model_executor/models/moss_audio.py +++ b/vllm/model_executor/models/moss_audio.py @@ -1356,32 +1356,17 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]): - def _apply_hf_processor_main( + def _call_hf_processor( self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - if not mm_data: - return BatchFeature(dict(passthrough_data)) - - mm_data = dict(mm_data) - audios = mm_data.pop("audios", []) - if audios: - mm_data["audio"] = audios - hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs) - processor_kwargs = _filter_moss_audio_processor_config(hf_processor_mm_kwargs) - processed_data = self.info.ctx.call_hf_processor( + processor_kwargs = _filter_moss_audio_processor_config(dict(hf_kwargs)) + return self.info.ctx.call_hf_processor( self.info.get_hf_processor(**processor_kwargs), - mm_data, + hf_data, {}, ) - processed_data.update(passthrough_data) - return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/moss_transcribe_diarize.py b/vllm/model_executor/models/moss_transcribe_diarize.py index 659b28d36b..32ac69c9af 100644 --- a/vllm/model_executor/models/moss_transcribe_diarize.py +++ b/vllm/model_executor/models/moss_transcribe_diarize.py @@ -66,7 +66,7 @@ from vllm.multimodal.processing import ( PromptUpdateDetails, cached_encode, ) -from vllm.multimodal.processing.processor import ProcessorInputs +from vllm.multimodal.processing.processor import HFMultiModalInputs, ProcessorInputs from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processor import cached_processor_from_config from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -465,17 +465,19 @@ class MossTranscribeDiarizeDummyInputsBuilder( class MossTranscribeDiarizeMultiModalProcessor( BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo] ): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _preprocess_hf_mm_data( + def _get_hf_mm_inputs( self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - audios = _get_audios_from_mm_data(mm_data) + mm_items: MultiModalDataItems, + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) + if hf_inputs.hf_data: + hf_inputs.hf_data["audio"] = _get_audios_from_mm_data(hf_inputs.hf_data) - return dict(audio=audios), hf_processor_mm_kwargs + return hf_inputs def _postprocess_hf_mm_data( self, diff --git a/vllm/model_executor/models/muse_glimmer.py b/vllm/model_executor/models/muse_glimmer.py index 192ac2de98..38cc475be2 100644 --- a/vllm/model_executor/models/muse_glimmer.py +++ b/vllm/model_executor/models/muse_glimmer.py @@ -275,16 +275,18 @@ class MuseGlimmerMultiModalProcessor( def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data) prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + processor = self.info.get_hf_processor(**hf_kwargs) tokenizer = processor.tokenizer config = self.info.get_hf_config() images = mm_data.get("images", ()) @@ -371,8 +373,9 @@ class MuseGlimmerMultiModalProcessor( video_feature_sizes=torch.tensor(video_sizes), ) processed_data = BatchFeature(data=data, tensor_type=None) - processed_data.update(passthrough_data) - return processed_data + return self._finalize_hf_mm_data( + mm_data, hf_kwargs, passthrough_data, processed_data + ) def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 2620923a49..eb5e9f06ca 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -70,6 +70,7 @@ from vllm.multimodal.processing import ( from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, + HFMultiModalInputs, PromptReplacement, PromptUpdate, cached_encode, @@ -196,6 +197,9 @@ NanoNemotronVLVideoInputs: TypeAlias = ( class NanoNemotronVLProcessingInfo(BaseProcessingInfo): def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor: + # Not accepted by NanoNemotronVLProcessor.__init__ + kwargs.pop("truncation", None) + return self.ctx.init_processor( NanoNemotronVLProcessor, config=self.get_hf_config(), @@ -358,9 +362,23 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo): class NanoNemotronVLMultiModalProcessor( BaseMultiModalProcessor[NanoNemotronVLProcessingInfo] ): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) + def _get_hf_mm_inputs( + self, + mm_items: MultiModalDataItems, + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) + + # NanoNemotronVLProcessor expects "audios" instead of "audio" + hf_data = hf_inputs.hf_data + if "audio" in hf_data: + hf_data["audios"] = hf_data.pop("audio") + + return hf_inputs + def _get_image_fields_config(self, hf_inputs: BatchFeature): if self.info.is_dynamic_tiler: pixel_values_flat = MultiModalFieldConfig.batched("image") @@ -730,15 +748,12 @@ class NanoNemotronVLMultiModalProcessor( # Bypass the cached path: the HF processor must receive the # prompt (with injected ) and the audio data # together so it can perform audio-token replacement natively. - mm_info = self._apply_hf_processor(inputs, timing_ctx) - prompt_ids = self._postprocess_prompt(inputs.prompt) + mm_res = self._apply_hf_processor(inputs, timing_ctx) with timing_ctx.record("apply_prompt_updates"): prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates( mm_items=mm_items, - prompt_ids=prompt_ids, - mm_kwargs=mm_info.kwargs, - mm_prompt_updates=mm_info.prompt_updates, + mm_res=mm_res, ) mm_placeholder_ranges = { @@ -749,8 +764,8 @@ class NanoNemotronVLMultiModalProcessor( return MultiModalInput( type="multimodal", prompt_token_ids=prompt_ids, - mm_kwargs=mm_info.kwargs, - mm_hashes=mm_info.hashes, + mm_kwargs=mm_res.kwargs, + mm_hashes=mm_res.hashes, mm_placeholders=mm_placeholder_ranges, ) diff --git a/vllm/model_executor/models/ovis.py b/vllm/model_executor/models/ovis.py index f5065659b4..5563b9a9ce 100644 --- a/vllm/model_executor/models/ovis.py +++ b/vllm/model_executor/models/ovis.py @@ -326,6 +326,9 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]): class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]): + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def image_indicators_to_visual_tokens( self, image_indicators: list[int], @@ -340,9 +343,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]): # -300 is image_atom token, filter them out return [vte_vocab_size + x + 300 for x in image_indicators if x < -300] - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: - return self.dummy_inputs.get_dummy_text(mm_counts) - def _postprocess_hf_mm_data( self, mm_data: Mapping[str, object], diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py index 607dcff894..0b8a2c5f84 100644 --- a/vllm/model_executor/models/ovis2_5.py +++ b/vllm/model_executor/models/ovis2_5.py @@ -327,7 +327,7 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]) if x >= INDICATOR_IDS[0] ] - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _postprocess_hf_mm_data( diff --git a/vllm/model_executor/models/paddleocr_vl.py b/vllm/model_executor/models/paddleocr_vl.py index d2587b0203..976d82032f 100644 --- a/vllm/model_executor/models/paddleocr_vl.py +++ b/vllm/model_executor/models/paddleocr_vl.py @@ -243,36 +243,37 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing class PaddleOCRVLMultiModalProcessor( BaseMultiModalProcessor[PaddleOCRVLProcessingInfo] ): - def _apply_hf_processor_main( + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _call_hf_processor( self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - if not mm_data: - return BatchFeature(dict(passthrough_data)) - - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - - final_mm_kwargs = dict(hf_processor_mm_kwargs or {}) + final_mm_kwargs = dict(hf_kwargs or {}) final_mm_kwargs.setdefault("images_kwargs", {}) # vLLM use PIL.Image, always set channel_last final_mm_kwargs["input_data_format"] = ChannelDimension.LAST - processed_data = self.info.ctx.call_hf_processor( + return self.info.ctx.call_hf_processor( self.info.get_hf_processor(**final_mm_kwargs), - dict(text=prompt_text, **mm_data), - hf_processor_mm_kwargs, + hf_data, + hf_kwargs, ) + + def _postprocess_hf_mm_data( + self, + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if not hf_data: + return processed_data + num_patches_per_image = processed_data["image_grid_thw"].prod(-1) processed_data["pixel_values"] = processed_data["pixel_values"].split( num_patches_per_image.tolist() ) - processed_data.update(passthrough_data) - return processed_data def _get_mm_fields_config( diff --git a/vllm/model_executor/models/phi3v.py b/vllm/model_executor/models/phi3v.py index 17477e8de1..a75b4afd22 100644 --- a/vllm/model_executor/models/phi3v.py +++ b/vllm/model_executor/models/phi3v.py @@ -390,28 +390,9 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]): class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _postprocess_hf_mm_data( - self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - processed_data: BatchFeature, - ) -> BatchFeature: - if not mm_data: - return processed_data - - input_ids = processed_data["input_ids"] - assert isinstance(input_ids, torch.Tensor) - - # Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids, - # which will cause OverflowError when decoding the prompt_ids. - # Therefore, we need to do an early replacement here - input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID) - - return processed_data - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/phi4mm.py b/vllm/model_executor/models/phi4mm.py index a08a17c324..c08eda612d 100644 --- a/vllm/model_executor/models/phi4mm.py +++ b/vllm/model_executor/models/phi4mm.py @@ -45,6 +45,7 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, + HFMultiModalInputs, PromptReplacement, PromptUpdate, ResolvedPromptUpdate, @@ -867,40 +868,38 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]): class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]): - def _apply_hf_processor_main( + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) + + # The Phi4MM processor expects "audios" instead of "audio" + hf_data = hf_inputs.hf_data + if "audio" in hf_data: + hf_data["audios"] = hf_data.pop("audio") + + sr = self.info.get_feature_extractor(**hf_inputs.hf_kwargs).sampling_rate + if audio_data := hf_data.get("audios", []): + assert isinstance(audio_data, Sequence) + hf_data["audios"] = [(data, sr) for data in audio_data] + + return hf_inputs + + def _postprocess_hf_mm_data( + self, + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], + processed_data: BatchFeature, ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + if not hf_data: + return processed_data - if not mm_data: - return BatchFeature(dict(passthrough_data)) - - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - - sr = self.info.get_feature_extractor(**hf_processor_mm_kwargs).sampling_rate - raw_audio_data = mm_data.get("audios", []) - assert isinstance(raw_audio_data, Sequence) - audio_data = [] - for data in raw_audio_data: - assert isinstance(data, (list, np.ndarray, torch.Tensor)) - audio_data.append(data) - - processor_data = dict(text=prompt_text, **mm_data) - if audio_data: - processor_data["audios"] = [(data, sr) for data in audio_data] - - processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - processor_data, - hf_processor_mm_kwargs, - ) - - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_kwargs) num_img_tokens = [ self.info.get_num_image_tokens( image_width=img_size[0], @@ -911,21 +910,26 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]): ] processed_data["num_img_tokens"] = num_img_tokens - audio_features = processed_data["input_audio_embeds"] + audios = hf_data.get("audios", []) + assert isinstance(audios, list) + audio_features = processed_data.get("input_audio_embeds") + if audio_features is None: + return processed_data + + sr = self.info.get_feature_extractor(**hf_kwargs).sampling_rate feature_sizes = [ - self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data + self.info.get_audio_num_frames(len(audio), sr) for audio, _ in audios ] processed_data["input_audio_embeds"] = [ audio_features[idx, :size] for idx, size in enumerate(feature_sizes) ] - processed_data.update(passthrough_data) return processed_data def _get_mm_fields_config( self, hf_inputs: BatchFeature, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> Mapping[str, MultiModalFieldConfig]: return dict( input_image_embeds=MultiModalFieldConfig.batched("image"), diff --git a/vllm/model_executor/models/phi4siglip.py b/vllm/model_executor/models/phi4siglip.py index 3c36b1f3ee..fd9fa94c7d 100644 --- a/vllm/model_executor/models/phi4siglip.py +++ b/vllm/model_executor/models/phi4siglip.py @@ -143,39 +143,8 @@ class Phi4SiglipDummyInputsBuilder( class Phi4SiglipMultiModalProcessor( BaseMultiModalProcessor[Phi4SiglipProcessingInfo], ): - def _apply_hf_processor_main( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - - processed = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - dict(text=prompt_text, **mm_data), - hf_processor_mm_kwargs, - ) - - # The HF processor's tokenizer_image_token() replaces the "" - # string with IMAGE_TOKEN_INDEX (-200) in input_ids. This breaks - # vLLM's prompt-replacement pipeline which needs to find "" - # as normal sub-tokens. Re-tokenize with the plain tokenizer so - # that "" stays as sub-tokens and can be located by - # PromptReplacement. - # NOTE: tokenizer.__call__() (not .encode()) must be used so that - # added/special tokens like <|user|>, <|end|> are kept as single IDs. - tokenizer = self.info.get_tokenizer() - new_ids = tokenizer(prompt_text).input_ids - processed["input_ids"] = torch.tensor([new_ids]) - - processed_data = processed - processed_data.update(passthrough_data) - return processed_data + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index 42812add9b..425bba74fd 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -14,7 +14,7 @@ from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk from mistral_common.protocol.instruct.messages import UserMessage from mistral_common.protocol.instruct.request import ChatCompletionRequest from packaging.version import Version -from transformers import BatchFeature, PixtralVisionConfig +from transformers import PixtralVisionConfig from transformers.models.pixtral.image_processing_pixtral import ( _num_image_tokens as _get_pixtral_hf_num_image_tokens, ) @@ -41,7 +41,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems from vllm.multimodal.inputs import ( MultiModalFieldConfig, MultiModalKwargsItem, - MultiModalKwargsOptionalItems, NestedTensors, ) from vllm.multimodal.parse import ( @@ -53,7 +52,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, - MultiModalPromptUpdates, + HFMultiModalInputs, + MultiModalProcessingResult, PlaceholderFeaturesInfo, ProcessorInputs, PromptReplacement, @@ -67,7 +67,6 @@ from vllm.transformers_utils.processors.pixtral import ( MistralCommonImageProcessor, MistralCommonPixtralProcessor, ) -from vllm.utils.collection_utils import is_list_of from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -268,21 +267,19 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]) def _maybe_apply_prompt_updates( self, mm_items: MultiModalDataItems, - prompt_ids: list[int], - mm_kwargs: MultiModalKwargsOptionalItems, - mm_prompt_updates: MultiModalPromptUpdates, + mm_res: MultiModalProcessingResult, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: mm_item_counts = mm_items.get_all_counts() - self._validate_mm_kwargs(mm_kwargs, mm_item_counts) - self._validate_mm_updates(mm_prompt_updates, mm_item_counts) + self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts) + self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts) mm_placeholders = self._find_mm_placeholders( - prompt_ids, - mm_prompt_updates, + mm_res.prompt_ids, + mm_res.prompt_updates, ) self._validate_mm_placeholders(mm_placeholders, mm_item_counts) - return prompt_ids, mm_placeholders + return mm_res.prompt_ids, mm_placeholders def _get_mm_fields_config( self, @@ -291,35 +288,17 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]) ) -> Mapping[str, MultiModalFieldConfig]: return dict(images=MultiModalFieldConfig.batched("image")) - def _apply_hf_processor_main( + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) + + # Avoid padding issue + return hf_inputs._replace( + hf_kwargs=dict(hf_inputs.hf_kwargs, return_tensors=None) ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - if not mm_data: - return BatchFeature(dict(passthrough_data)) - - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - - outputs = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - dict(text=prompt_text, **mm_data), - # Avoid padding issue - dict(**hf_processor_mm_kwargs, return_tensors=None), - ) - - # Missing batch dimension - if is_list_of(outputs["input_ids"], int): - outputs["input_ids"] = [outputs["input_ids"]] - - processed_data = outputs - processed_data.update(passthrough_data) - return processed_data def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index bb6b153edd..cecc8345e5 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -82,6 +82,8 @@ from vllm.multimodal.processing import ( ) from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, + HFMultiModalInputs, + MultiModalProcessingResult, MultiModalPromptUpdates, PlaceholderFeaturesInfo, PromptReplacement, @@ -498,6 +500,64 @@ class Qwen2_5OmniThinkerDummyInputsBuilder( class Qwen2_5OmniThinkerMultiModalProcessor( BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo] ): + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _get_hf_mm_inputs( + self, + mm_items: MultiModalDataItems, + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) + + mm_counts = mm_items.get_all_counts() + if hf_kwargs.get("use_audio_in_video"): + num_videos = mm_counts.get("video", 0) + if mm_counts.get("audio", 0) < num_videos: + raise ValueError( + "Video doesn't have audio track with `audio_in_video=True`" + ) + + if num_videos and "text" in hf_inputs.hf_data: + standalone_mm_counts = dict(mm_counts) + standalone_mm_counts["audio"] -= num_videos + hf_inputs.hf_data["text"] = self._get_hf_mm_text(standalone_mm_counts) + + hf_data = hf_inputs.hf_data + normalized_kwargs = hf_inputs.hf_kwargs + audios = hf_data.pop("audio", []) + + # NOTE: WhisperFeatureExtractor cannot handle empty list of audios + if audios: + hf_data["audio"] = audios + elif ( + normalized_kwargs.get("use_audio_in_video") + and "audio" not in hf_data + and hf_data.get("videos") + ): + raise ValueError( + "Video doesn't have audio track with `audio_in_video=True`" + ) + + merged = self.info.ctx.get_merged_mm_kwargs(normalized_kwargs) + if hf_data.get("videos") and ( + merged.keys() & {"size", "min_pixels", "max_pixels"} + ): + normalized_kwargs = dict(normalized_kwargs) + video_size = dict(self.info.get_hf_processor().video_processor.size) + size_override = merged.get("size") + if size_override is not None: + video_size = video_size | size_override + min_pixels = merged.get("min_pixels") + if min_pixels is not None: + video_size["shortest_edge"] = min_pixels + max_pixels = merged.get("max_pixels") + if max_pixels is not None: + video_size["longest_edge"] = max_pixels + normalized_kwargs["size"] = video_size + + return hf_inputs._replace(hf_kwargs=normalized_kwargs) + def _get_mm_fields_config( self, hf_inputs: BatchFeature, @@ -570,12 +630,14 @@ class Qwen2_5OmniThinkerMultiModalProcessor( def _maybe_apply_prompt_updates( self, mm_items: MultiModalDataItems, - prompt_ids: list[int], - mm_kwargs: MultiModalKwargsItems, - mm_prompt_updates: MultiModalPromptUpdates, + mm_res: MultiModalProcessingResult, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: """Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.""" mm_item_counts = mm_items.get_all_counts() + prompt_ids = mm_res.prompt_ids + mm_kwargs = mm_res.kwargs + mm_prompt_updates = mm_res.prompt_updates + self._validate_mm_kwargs(mm_kwargs, mm_item_counts) self._validate_mm_updates(mm_prompt_updates, mm_item_counts) @@ -789,101 +851,33 @@ class Qwen2_5OmniThinkerMultiModalProcessor( ), ] - def _apply_hf_processor_main( + def _postprocess_hf_mm_data( self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], + processed_data: BatchFeature, ) -> BatchFeature: - """Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.""" - mm_counts = mm_items.get_all_counts() - - use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False) - if use_audio_in_video and "video" in mm_counts: - if mm_counts.get("audio", 0) < mm_counts["video"]: - raise ValueError( - "Video doesn't have audio track with `audio_in_video=True`" - ) - mm_counts["audio"] -= mm_counts["video"] - - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - prompt_text = self.dummy_inputs.get_dummy_text(mm_counts) - - mm_data = dict(processor_data) - audios = mm_data.pop("audios", []) - - # NOTE: WhisperFeatureExtractor cannot handle empty list of audios - if audios: - # NOTE: Qwen2.5-Omni processor accept "audio" - mm_data["audio"] = audios - hf_processor_mm_kwargs = dict( - **hf_processor_mm_kwargs, - ) - elif ( - hf_processor_mm_kwargs.get("use_audio_in_video") - and "audio" not in mm_data - and mm_data.get("videos") - ): - # A subclass (e.g. Qwen3-Omni) may have already popped "audios" - # and populated mm_data["audio"] itself before delegating here, - # in which case this isn't a genuine "no audio" case. Likewise, - # mm_data can be empty on a multimodal-processor-cache hit, where - # there's nothing to (re-)process this call and the real result - # comes from the cache — not a genuine "no audio" case either. - raise ValueError( - "Video doesn't have audio track with `audio_in_video=True`" - ) - - merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs) - if mm_data.get("videos") and ( - merged.keys() & {"size", "min_pixels", "max_pixels"} - ): - hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs) - video_size = dict(self.info.get_hf_processor().video_processor.size) - size_override = merged.get("size") - if size_override is not None: - video_size = video_size | size_override - min_pixels = merged.get("min_pixels") - if min_pixels is not None: - video_size["shortest_edge"] = min_pixels - max_pixels = merged.get("max_pixels") - if max_pixels is not None: - video_size["longest_edge"] = max_pixels - hf_processor_mm_kwargs["size"] = video_size - - hf_inputs = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - dict(text=prompt_text, **mm_data), - hf_processor_mm_kwargs, - ) - - input_features = hf_inputs.pop("input_features", None) - feature_attention_mask = hf_inputs.get("feature_attention_mask", None) - if "input_audio_features" not in hf_inputs and input_features is not None: + input_features = processed_data.pop("input_features", None) + feature_attention_mask = processed_data.get("feature_attention_mask", None) + if "input_audio_features" not in processed_data and input_features is not None: if feature_attention_mask is not None: input_features = input_features.permute(0, 2, 1)[ feature_attention_mask.bool() ].permute(1, 0) - hf_inputs["input_audio_features"] = input_features + processed_data["input_audio_features"] = input_features if ( - "audio_feature_lengths" not in hf_inputs + "audio_feature_lengths" not in processed_data and feature_attention_mask is not None ): - hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1) + processed_data["audio_feature_lengths"] = feature_attention_mask.sum(-1) - video_second_per_grid = hf_inputs.get("video_second_per_grid", None) + video_second_per_grid = processed_data.get("video_second_per_grid", None) if video_second_per_grid is not None: - hf_inputs["second_per_grid_ts"] = video_second_per_grid + processed_data["second_per_grid_ts"] = video_second_per_grid - use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False) - hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video) - - processed_data = hf_inputs - processed_data.update(passthrough_data) - processed_data.pop("input_ids") + use_audio_in_video = hf_kwargs.get("use_audio_in_video", False) + processed_data["use_audio_in_video"] = torch.tensor(use_audio_in_video) + processed_data.pop("input_ids", None) return processed_data diff --git a/vllm/model_executor/models/qwen2_audio.py b/vllm/model_executor/models/qwen2_audio.py index 27c603c866..7e7ffd5cfa 100644 --- a/vllm/model_executor/models/qwen2_audio.py +++ b/vllm/model_executor/models/qwen2_audio.py @@ -59,6 +59,7 @@ from vllm.multimodal.processing import ( PromptReplacement, PromptUpdate, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.sequence import IntermediateTensors from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -233,29 +234,25 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _preprocess_hf_mm_data( + def _get_hf_mm_inputs( self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + mm_items: MultiModalDataItems, + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) - mm_data = dict(mm_data) - audios = mm_data.pop("audios", []) - if audios: - mm_data["audio"] = audios - - hf_processor_mm_kwargs = dict( - **hf_processor_mm_kwargs, - sampling_rate=feature_extractor.sampling_rate, - truncation=True, + feature_extractor = self.info.get_feature_extractor(**hf_kwargs) + return hf_inputs._replace( + hf_kwargs=dict( + hf_inputs.hf_kwargs, + sampling_rate=feature_extractor.sampling_rate, + truncation=True, + ) ) - return mm_data, hf_processor_mm_kwargs - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/qwen3_asr_realtime.py b/vllm/model_executor/models/qwen3_asr_realtime.py index 15ecd861f1..bab689c605 100644 --- a/vllm/model_executor/models/qwen3_asr_realtime.py +++ b/vllm/model_executor/models/qwen3_asr_realtime.py @@ -37,11 +37,10 @@ from vllm.model_executor.models.qwen3_asr import ( ) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache -from vllm.multimodal.inputs import MultiModalKwargsOptionalItems from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing.processor import ( BaseDummyInputsBuilder, - MultiModalPromptUpdates, + MultiModalProcessingResult, PlaceholderFeaturesInfo, cached_encode, ) @@ -114,10 +113,10 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor): def _maybe_apply_prompt_updates( self, mm_items: MultiModalDataItems, - prompt_ids: list[int], - mm_kwargs: MultiModalKwargsOptionalItems, - mm_prompt_updates: MultiModalPromptUpdates, + mm_res: MultiModalProcessingResult, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: + mm_kwargs = mm_res.kwargs + audios = mm_kwargs.get("audio", []) assert len(audios) == 1, ( f"Expected only one audio input for realtime, got {len(audios)}" @@ -147,7 +146,7 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor): # Find the audio_pad token position and expand it to audio_len tokens expanded_ids = list[int]() pad_start_idx = -1 - for i, tid in enumerate(prompt_ids): + for i, tid in enumerate(mm_res.prompt_ids): if tid == audio_pad_id and pad_start_idx == -1: pad_start_idx = i expanded_ids.extend([audio_pad_id] * audio_len) diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index ce800e7de6..2c3647c81e 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -73,6 +73,8 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems from vllm.multimodal.processing.processor import ( + HFMultiModalInputs, + MultiModalProcessingResult, MultiModalPromptUpdates, PlaceholderFeaturesInfo, PromptReplacement, @@ -1219,13 +1221,26 @@ Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder class Qwen3OmniMoeThinkerMultiModalProcessor( Qwen2_5OmniThinkerMultiModalProcessor, ): - def _get_hf_mm_data( + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - processor_data, passthrough_data = super()._get_hf_mm_data(mm_items) + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) - audios = processor_data.get("audios") + if mm_items.get_count("audio", strict=False): + # TODO(Isotr0py): Remove this patch after upstream fix PR + # released and Transformers version update: + # https://github.com/huggingface/transformers/pull/41473 + hf_inputs = hf_inputs._replace( + hf_kwargs={ + **hf_inputs.hf_kwargs, + "audio_kwargs": dict(hf_inputs.hf_kwargs.get("audio_kwargs") or {}), + "text_kwargs": dict(hf_inputs.hf_kwargs.get("text_kwargs") or {}), + } + ) + + audios = hf_inputs.hf_data.get("audio") if audios: feature_extractor = self.info.get_feature_extractor() hop_length = feature_extractor.hop_length @@ -1239,47 +1254,30 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( # To make sure the cache works with padding=True, we pre-padded # the audio to a multiple of hop_length. - processor_data = dict(processor_data) - processor_data["audios"] = [ + hf_inputs.hf_data["audio"] = [ pad_to_hop_length(audio) if isinstance(audio, np.ndarray) else (pad_to_hop_length(audio[0]), audio[1]) for audio in audios ] - return processor_data, passthrough_data + return hf_inputs - def _apply_hf_processor_main( + def _postprocess_hf_mm_data( self, - mm_items: MultiModalDataItems, + mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, ) -> BatchFeature: - if mm_items.get_all_counts().get("audio", 0): - # TODO(Isotr0py): Remove this patch after upstream fix PR - # released and Transformers version update: - # https://github.com/huggingface/transformers/pull/41473 - hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs) - hf_processor_mm_kwargs["audio_kwargs"] = dict( - hf_processor_mm_kwargs.get("audio_kwargs") or {} - ) - hf_processor_mm_kwargs["text_kwargs"] = dict( - hf_processor_mm_kwargs.get("text_kwargs") or {} - ) - - processed_data = super()._apply_hf_processor_main( - mm_items, - hf_processor_mm_kwargs, + processed_data = super()._postprocess_hf_mm_data( + mm_data, hf_processor_mm_kwargs, processed_data ) if ( "audio_feature_lengths" in processed_data and "feature_attention_mask" in processed_data ): - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - processor_data, _ = self._get_hf_mm_data(valid_mm_items) - audios = processor_data.get("audios", []) + audios = mm_data.get("audio", []) if audios: feature_extractor = self.info.get_feature_extractor( **hf_processor_mm_kwargs @@ -1312,12 +1310,14 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( def _maybe_apply_prompt_updates( self, mm_items: MultiModalDataItems, - prompt_ids: list[int], - mm_kwargs: MultiModalKwargsItems, - mm_prompt_updates: MultiModalPromptUpdates, + mm_res: MultiModalProcessingResult, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: """Qwen3-Omni reimplements this function to handle `use_audio_in_video`.""" mm_item_counts = mm_items.get_all_counts() + prompt_ids = mm_res.prompt_ids + mm_kwargs = mm_res.kwargs + mm_prompt_updates = mm_res.prompt_updates + self._validate_mm_kwargs(mm_kwargs, mm_item_counts) use_audio_in_video = False diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 482fd00e5d..78dda9c815 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -1327,27 +1327,25 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) proc_impl = getattr(type(hf_processor), "replace_video_token", None) return proc_impl is not None and proc_impl is not mixin_impl + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - if not mm_data: - return BatchFeature(dict(passthrough_data)) - - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - - mm_data = dict(mm_data) + if not hf_data: + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data) # Separate video processing from image processing. Because the videos # are processed into several image patches video_input_ids_lst: list[list[int]] = [] - if videos := mm_data.pop("videos", []): + if videos := hf_data.pop("videos", []): video_grid_thw_lst = [] pixel_values_videos_lst = [] timestamps_per_video = [] @@ -1370,10 +1368,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # NOTE: a copy of is created to update do_sample_frames, # otherwise mm_hash for the object will be incorrect. - video_mm_kwargs = dict(**hf_processor_mm_kwargs) - merged = self.info.ctx.get_merged_mm_kwargs( - hf_processor_mm_kwargs, modality="video" - ) + video_mm_kwargs = dict(**hf_kwargs) + merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video") if merged.keys() & {"size", "min_pixels", "max_pixels"}: video_size = dict(self.info.get_video_processor().size) size_override = merged.get("size") @@ -1485,13 +1481,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # fps/num_frames are video-only kwargs already consumed by the loop; # exclude them so the text/image processor call below never gets a list. non_video_mm_kwargs = { - k: v - for k, v in hf_processor_mm_kwargs.items() - if k not in ("fps", "num_frames") + k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames") } processed_data = self.info.ctx.call_hf_processor( self.info.get_hf_processor(**non_video_mm_kwargs), - dict(text=prompt_text, **mm_data), + hf_data, non_video_mm_kwargs, ) @@ -1516,9 +1510,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) processed_data["input_ids"] = [expanded_ids] processed_data.update(video_outputs) - processed_data.update(passthrough_data) - - return processed_data + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data + ) def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/terratorch.py b/vllm/model_executor/models/terratorch.py index 640f946476..ddd1ac81ce 100644 --- a/vllm/model_executor/models/terratorch.py +++ b/vllm/model_executor/models/terratorch.py @@ -202,7 +202,9 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs with timing_ctx.record("apply_hf_processor"): - _, passthrough_data = self._get_hf_mm_data(mm_items) + passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_processor_mm_kwargs + ).passthrough_data mm_processed_data = BatchFeature( { k: torch.as_tensor(v).unsqueeze(0) diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index ff042e2f92..7b435d40ba 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -219,18 +219,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo] Subclasses add the strategy for locating placeholders in the prompt. """ - def _get_hf_mm_data( - self, - mm_items: MultiModalDataItems, - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - """Rename the parser's `audios` key to the `audio` argument HF audio - processors take. - """ - processor_data, passthrough_data = super()._get_hf_mm_data(mm_items) - if self.info._is_audio_model() and "audios" in processor_data: - processor_data["audio"] = processor_data.pop("audios") - return processor_data, passthrough_data - def _get_modality_field_names(self, modality: str) -> set[str]: """Names of the fields the sub-processor for `modality` produces.""" # TODO: use else branch only once huggingface/transformers#44394 lands. @@ -353,6 +341,9 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo] return mm_fields + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _unpad_audios( self, hf_inputs: "BatchFeature", @@ -597,21 +588,31 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase): # work for Transformers. The vision path has logic tied to # `mm_tokens_per_modality` in _apply_vision() hf_processor_mm_kwargs = { - **hf_processor_mm_kwargs, + # vLLM needs the untruncated sequence to keep placeholder + # tokens aligned. Note that the text inputs are just dummy + # text, not the original prompt. The original prompt is + # already tokenized by the renderer. + "truncation": False, # HF processors only accept text, and the decoded string already # contains any special tokens, so don't let them be added again # (the HF processor call disables `add_special_tokens`). "add_special_tokens": False, + **hf_processor_mm_kwargs, } - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + processor_data, _, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_processor_mm_kwargs ) - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + # The real prompt is fed to the processor below and the placeholder + # ranges are read back out of its output, so the dummy text meant for + # media-only callers is discarded here. + processor_data.pop("text", None) + has_mm_data = any(processor_data.values()) # Ask which modality owns each token of the expanded prompt, which # is the only marking of the tokens an expansion adds around an item - if any(processor_data.values()): + if has_mm_data: processor_data = {**processor_data, "return_mm_token_type_ids": True} try: @@ -621,7 +622,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase): hf_processor_mm_kwargs, ) except ValueError: - if any(processor_data.values()): + if has_mm_data: raise # Some processors reject a prompt holding placeholders with # no data to go with them, so tokenize it without them @@ -792,31 +793,37 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): except (AttributeError, KeyError, TypeError): return None + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> "BatchFeature": - mm_counts = mm_items.get_all_counts() + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs + ) - valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0}) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + if not hf_data: + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data) - prompt_text = self.dummy_inputs.get_dummy_text(mm_counts) + prompt_text = hf_data.pop("text") + assert isinstance(prompt_text, str) # Ask for the replacement each placeholder expands to, and record it as # per-item fields: its token ids, and the tokens or patches behind them - if has_mm_data := any(mm_data.values()): - mm_data = {**mm_data, "return_text_replacement_offsets": True} + if has_mm_data := any(hf_data.values()): + hf_data = {**hf_data, "return_text_replacement_offsets": True} try: hf_inputs = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - dict(text=prompt_text, **mm_data), - hf_processor_mm_kwargs, + self.info.get_hf_processor(**hf_kwargs), + dict(text=prompt_text, **hf_data), + hf_kwargs, ) except ValueError: - if any(mm_data.values()): + if any(hf_data.values()): raise # Some processors reject a prompt holding placeholders with # no data to go with them, so tokenize it without them @@ -825,7 +832,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt" ) self._unpad_images(hf_inputs) - self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs) + self._unpad_audios(hf_inputs, hf_data, hf_kwargs) # Drop the inputs the model would reject hf_inputs.pop("mm_token_type_ids", None) @@ -845,9 +852,10 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): "so it can be fixed, and install `transformers<5.15.0` in the " "meantime to locate placeholders in the expanded prompt instead." ) - hf_inputs.update(passthrough_data) hf_inputs.pop("input_ids", None) - return hf_inputs + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, hf_inputs + ) tokenizer = self.info.get_tokenizer() replacements = defaultdict[str, list[list[int]]](list) @@ -865,7 +873,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): ) if modality == "image": hf_inputs["num_image_patches"] = self._get_num_image_patches( - hf_inputs, mm_data, len(seqs) + hf_inputs, hf_data, len(seqs) ) elif modality == "audio": counts = [] @@ -874,10 +882,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): counts.append(int(ids.eq(_get_embed_token_id(ids)).sum())) hf_inputs["num_audio_tokens"] = torch.tensor(counts) - hf_inputs.update(passthrough_data) hf_inputs.pop("input_ids") - return hf_inputs + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, hf_inputs + ) # From this version on, a processor reporting no offsets is an error rather than a diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index 6465972cf1..3a1d017f4c 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -42,6 +42,7 @@ from vllm.multimodal.processing import ( PromptUpdate, cached_encode, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.ultravox import UltravoxConfig @@ -190,72 +191,32 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]) class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]): - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _preprocess_hf_mm_data( - self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - audios = mm_data.get("audios", []) - assert isinstance(audios, list) - - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) - hf_processor_mm_kwargs = dict( - **hf_processor_mm_kwargs, - sampling_rate=feature_extractor.sampling_rate, - include_audio_num_chunks=True, - ) - - return mm_data, hf_processor_mm_kwargs - - def _apply_hf_processor_main( + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) - if processor_data: - processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data( - processor_data, - hf_processor_mm_kwargs, + # The Ultravox processor accepts "audios" instead of "audio" + hf_data = hf_inputs.hf_data + if "audio" in hf_data: + hf_data["audios"] = hf_data.pop("audio") + + feature_extractor = self.info.get_feature_extractor(**hf_kwargs) + normalized_kwargs = dict(hf_inputs.hf_kwargs) + # The remote processor already hardcodes truncation=False and forwards + # its kwargs to the audio processor, where another value would collide. + normalized_kwargs.pop("truncation", None) + return hf_inputs._replace( + hf_kwargs=dict( + normalized_kwargs, + sampling_rate=feature_extractor.sampling_rate, + include_audio_num_chunks=True, ) - - prompt_text = self._get_hf_processor_text(mm_items.get_all_counts()) - if prompt_text is not None: - processor_data = dict(text=prompt_text, **processor_data) - - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - - def patched_call(**kwargs) -> BatchFeature: - # The remote UltravoxProcessor does not support the - # `truncation` kwarg injected by `call_hf_processor`: it - # passes `truncation` to its audio processor by itself while - # also forwarding `**kwargs` there, causing a duplicate - # keyword argument error. Its hardcoded `truncation=False` - # already matches vLLM's intended behavior. - kwargs.pop("truncation", None) - - return hf_processor(**kwargs) - - processed_data = self.info.ctx.call_hf_processor( - patched_call, - processor_data, - hf_processor_mm_kwargs, - ) - processed_data.update(passthrough_data) - else: - processed_data = BatchFeature(dict(passthrough_data)) - - return self._postprocess_hf_mm_data( - processor_data, - hf_processor_mm_kwargs, - processed_data, ) def _postprocess_hf_mm_data( diff --git a/vllm/model_executor/models/unlimited_ocr.py b/vllm/model_executor/models/unlimited_ocr.py index c96e5f293d..d248e5c8e0 100644 --- a/vllm/model_executor/models/unlimited_ocr.py +++ b/vllm/model_executor/models/unlimited_ocr.py @@ -75,7 +75,7 @@ from vllm.multimodal.parse import ( from vllm.multimodal.processing import PromptReplacement, PromptUpdate from vllm.multimodal.processing.context import TimingContext from vllm.multimodal.processing.inputs import ProcessorInputs -from vllm.multimodal.processing.processor import MultiModalProcessingInfo +from vllm.multimodal.processing.processor import MultiModalProcessingResult from vllm.transformers_utils.processors.deepseek_ocr import ( BASE_SIZE, CROP_MODE, @@ -225,7 +225,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor): self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> MultiModalProcessingInfo: + ) -> MultiModalProcessingResult: # The processor logic differs for single-image (crop) vs multi-image # (no crop) requests. The processing cache assumes per-item output is # invariant of how many images are passed per prompt, so we only cache diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index 5559217d37..ad54e8eb20 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -16,7 +16,7 @@ from mistral_common.protocol.instruct.messages import UserMessage from mistral_common.protocol.instruct.request import ChatCompletionRequest from mistral_common.protocol.transcription.request import TranscriptionRequest from mistral_common.tokens.tokenizers.audio import Audio -from transformers import BatchFeature, WhisperConfig +from transformers import WhisperConfig from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import BaseDummyOptions @@ -36,7 +36,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, MultiModalKwargsItems, - MultiModalKwargsOptionalItems, NestedTensors, ) from vllm.multimodal.parse import ( @@ -48,7 +47,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, - MultiModalPromptUpdates, + HFMultiModalInputs, + MultiModalProcessingResult, PlaceholderFeaturesInfo, ProcessorInputs, PromptReplacement, @@ -61,7 +61,6 @@ from vllm.transformers_utils.processors.voxtral import ( MistralCommonFeatureExtractor, MistralCommonVoxtralProcessor, ) -from vllm.utils.collection_utils import is_list_of from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription from .utils import init_vllm_registered_model, maybe_prefix @@ -209,21 +208,19 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]) def _maybe_apply_prompt_updates( self, mm_items: MultiModalDataItems, - prompt_ids: list[int], - mm_kwargs: MultiModalKwargsOptionalItems, - mm_prompt_updates: MultiModalPromptUpdates, + mm_res: MultiModalProcessingResult, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: mm_item_counts = mm_items.get_all_counts() - self._validate_mm_kwargs(mm_kwargs, mm_item_counts) - self._validate_mm_updates(mm_prompt_updates, mm_item_counts) + self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts) + self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts) mm_placeholders = self._find_mm_placeholders( - prompt_ids, - mm_prompt_updates, + mm_res.prompt_ids, + mm_res.prompt_updates, ) self._validate_mm_placeholders(mm_placeholders, mm_item_counts) - return prompt_ids, mm_placeholders + return mm_res.prompt_ids, mm_placeholders def _get_mm_fields_config( self, @@ -241,38 +238,20 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]) # skip validation here pass - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) - def _preprocess_hf_mm_data( + def _get_hf_mm_inputs( self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - mm_data = dict(mm_data) - audios = mm_data.pop("audios", []) - - if audios: - # MistralCommonVoxtralProcessor accepts "audio" - mm_data["audio"] = audios + mm_items: MultiModalDataItems, + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) # Avoid padding issue - hf_processor_mm_kwargs = dict(**hf_processor_mm_kwargs, return_tensors=None) - - return mm_data, hf_processor_mm_kwargs - - def _postprocess_hf_mm_data( - self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - processed_data: BatchFeature, - ) -> BatchFeature: - # Missing batch dimension - input_ids = processed_data.get("input_ids") - if is_list_of(input_ids, int): - processed_data["input_ids"] = [input_ids] - - return processed_data + return hf_inputs._replace( + hf_kwargs=dict(hf_inputs.hf_kwargs, return_tensors=None) + ) def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/voxtral_realtime.py b/vllm/model_executor/models/voxtral_realtime.py index 52b636cc25..162fecf038 100644 --- a/vllm/model_executor/models/voxtral_realtime.py +++ b/vllm/model_executor/models/voxtral_realtime.py @@ -28,11 +28,10 @@ from vllm.model_executor.models.voxtral import ( ) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache -from vllm.multimodal.inputs import MultiModalKwargsOptionalItems from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import BaseDummyInputsBuilder from vllm.multimodal.processing.processor import ( - MultiModalPromptUpdates, + MultiModalProcessingResult, PlaceholderFeaturesInfo, ) from vllm.sequence import IntermediateTensors @@ -60,10 +59,10 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor): def _maybe_apply_prompt_updates( self, mm_items: MultiModalDataItems, - prompt_ids: list[int], - mm_kwargs: MultiModalKwargsOptionalItems, - mm_prompt_updates: MultiModalPromptUpdates, + mm_res: MultiModalProcessingResult, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: + mm_kwargs = mm_res.kwargs + # there are no placeholder audio tokens for streaming # so we need to build the place placeholder positions manually @@ -86,7 +85,7 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor): * [0], # only used for length computation, so we can take dummy inputs is_embed=None, ) - return prompt_ids, {"audio": [features_info]} + return mm_res.prompt_ids, {"audio": [features_info]} class TimeEmbedding(torch.nn.Module): diff --git a/vllm/model_executor/models/whisper.py b/vllm/model_executor/models/whisper.py index 7fc620e7dd..b50d04d04b 100644 --- a/vllm/model_executor/models/whisper.py +++ b/vllm/model_executor/models/whisper.py @@ -61,6 +61,7 @@ from vllm.multimodal.processing import ( PromptReplacement, PromptUpdate, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.renderers import TokenizeParams from vllm.transformers_utils.processor import cached_processor_from_config from vllm.utils.jsontree import json_map_leaves @@ -743,6 +744,9 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]): class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo]): + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def create_encoder_prompt( self, prompt: list[int], @@ -754,37 +758,21 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo # for encoder profiling. return [0] - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: - return self.dummy_inputs.get_dummy_text(mm_counts) - - def _preprocess_hf_mm_data( + def _get_hf_mm_inputs( self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + mm_items: MultiModalDataItems, + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) - mm_data = dict(mm_data) - mm_data["audio"] = mm_data.pop("audios") - - hf_processor_mm_kwargs = dict( - **hf_processor_mm_kwargs, - sampling_rate=feature_extractor.sampling_rate, + feature_extractor = self.info.get_feature_extractor(**hf_kwargs) + return hf_inputs._replace( + hf_kwargs=dict( + hf_inputs.hf_kwargs, + sampling_rate=feature_extractor.sampling_rate, + ) ) - return mm_data, hf_processor_mm_kwargs - - def _postprocess_hf_mm_data( - self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - processed_data: BatchFeature, - ) -> BatchFeature: - if "labels" in processed_data: - processed_data["input_ids"] = processed_data.pop("labels") - - return processed_data - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/models/dots3_note/common/processor.py b/vllm/models/dots3_note/common/processor.py index e8d365aa38..addaf77c81 100644 --- a/vllm/models/dots3_note/common/processor.py +++ b/vllm/models/dots3_note/common/processor.py @@ -34,6 +34,7 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.transformers_utils.repo_utils import get_hf_file_to_dict from .video import preprocess_dots3_note_video @@ -630,26 +631,33 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingInfo]): - def _get_hf_mm_data( + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - processor_data, passthrough_data = super()._get_hf_mm_data(mm_items) - if "video" not in mm_items: - return processor_data, passthrough_data + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) - videos = mm_items.get_items("video", VideoProcessorItems) - raw_videos: list[object] = [] - for index, item in enumerate(videos.data): - if isinstance(item, MediaWithBytes): - raw_videos.append(item.original_bytes) - else: - raw_videos.append(videos.get(index)) - processor_data = dict(processor_data) - processor_data["videos"] = raw_videos - return processor_data, passthrough_data + # The Dots3Note processor accepts "audios" instead of "audio" + hf_data = hf_inputs.hf_data + if "audio" in hf_data: + hf_data["audios"] = hf_data.pop("audio") - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + if "video" in mm_items: + videos = mm_items.get_items("video", VideoProcessorItems) + + raw_videos: list[object] = [] + for index, item in enumerate(videos.data): + if isinstance(item, MediaWithBytes): + raw_videos.append(item.original_bytes) + else: + raw_videos.append(videos.get(index)) + + hf_data["videos"] = raw_videos + + return hf_inputs + + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _get_mm_fields_config( diff --git a/vllm/models/inkling/common/mm_preprocess.py b/vllm/models/inkling/common/mm_preprocess.py index 249d7edeb4..cc88c12529 100644 --- a/vllm/models/inkling/common/mm_preprocess.py +++ b/vllm/models/inkling/common/mm_preprocess.py @@ -182,20 +182,19 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]) def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + processor = self.info.get_hf_processor(**hf_kwargs) tokenizer = self.info.get_tokenizer() images = mm_data.get("images") or [] - audios = mm_data.get("audios") or [] + audios = mm_data.get("audio") or [] if not isinstance(images, list): images = list(cast(Iterable[Any], images)) if not isinstance(audios, list): @@ -235,8 +234,9 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]) data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64) processed_data = BatchFeature(data=data, tensor_type=None) - processed_data.update(passthrough_data) - return processed_data + return self._finalize_hf_mm_data( + mm_data, hf_kwargs, passthrough_data, processed_data + ) def _tokenize_with_placeholders( self, diff --git a/vllm/models/minimax_m3/common/mm_preprocess.py b/vllm/models/minimax_m3/common/mm_preprocess.py index 9f1d08011a..4e8b791093 100644 --- a/vllm/models/minimax_m3/common/mm_preprocess.py +++ b/vllm/models/minimax_m3/common/mm_preprocess.py @@ -32,6 +32,7 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) +from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.multimodal.video import ( VIDEO_LOADER_REGISTRY, VideoBackend, @@ -307,26 +308,21 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing class MiniMaxM3VLMultiModalProcessor( BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo] ): - def _apply_hf_processor_main( + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) + hf_data = hf_inputs.hf_data - if not mm_data: - return BatchFeature(dict(passthrough_data)) - - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - - mm_data = dict(mm_data) # With ``video_needs_metadata=True`` each video arrives as a # ``(frames, metadata)`` tuple. Split the frames back out and forward the # metadata as ``VideoMetadata`` so the processor emits timestamps. - videos = cast(list | None, mm_data.get("videos")) + videos = cast(list | None, hf_data.get("videos")) video_metadata: list[VideoMetadata] | None = None if videos: frames_only = [] @@ -346,21 +342,25 @@ class MiniMaxM3VLMultiModalProcessor( # stays consistent with _get_prompt_updates. meta.setdefault("total_num_frames", len(frames)) video_metadata.append(VideoMetadata(**meta)) - mm_data["videos"] = frames_only + hf_data["videos"] = frames_only - # Override the video processor's default do_resize=False (set for a - # pre-resized pipeline) to True for vLLM's raw-frame inputs. - merged = dict(do_resize=True, **hf_processor_mm_kwargs) - data = dict(text=prompt_text, **mm_data) if video_metadata is not None: - data["video_metadata"] = video_metadata - processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - data, + hf_data["video_metadata"] = video_metadata + + return hf_inputs + + def _call_hf_processor( + self, + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], + ) -> BatchFeature: + # Override the video processor's default for vLLM's raw-frame inputs. + merged = dict(do_resize=True, **hf_kwargs) + return self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_kwargs), + hf_data, merged, ) - processed_data.update(passthrough_data) - return processed_data def _get_mm_fields_config( self, diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py index c1b81a4d72..359c43700d 100644 --- a/vllm/multimodal/processing/context.py +++ b/vllm/multimodal/processing/context.py @@ -321,12 +321,6 @@ class InputProcessingContext: merged_kwargs = self.get_merged_mm_kwargs(kwargs) - # vLLM needs the full untruncated sequence to keep multi-modal - # placeholder tokens aligned; note that the text inputs in - # call_hf_processor are just dummy text, not the original prompt. - # The original prompt is already tokenized by the renderer. - merged_kwargs.setdefault("truncation", False) - allowed_kwargs = get_allowed_kwarg_only_overrides( hf_processor, merged_kwargs, @@ -426,7 +420,7 @@ class BaseProcessingInfo: def get_data_parser(self) -> MultiModalDataParser: """Constructs a parser to preprocess multi-modal data items before passing them to - [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data]. + [`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs]. You can support additional modalities by creating a subclass of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser] @@ -506,7 +500,7 @@ class BaseProcessingInfo: """Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict] to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems] before passing them to - [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data]. + [`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs]. """ mm_items = self.data_parser.parse_mm_data(mm_data) diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index 8e9983798f..1efe84219e 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -1008,7 +1008,20 @@ For an item `MultiModalPromptUpdates[k][i]`, _I = TypeVar("_I", bound=BaseProcessingInfo) -class MultiModalProcessingInfo(NamedTuple): +class HFMultiModalInputs(NamedTuple): + hf_data: dict[str, object] + """The data inputs to the HF processor.""" + + hf_kwargs: Mapping[str, object] + """The non-data inputs to the HF processor.""" + + passthrough_data: dict[str, object] + """The inputs to assign to the output of the HF processor, + instead of passing them to the HF processor.""" + + +class MultiModalProcessingResult(NamedTuple): + prompt_ids: list[int] kwargs: MultiModalKwargsOptionalItems hashes: MultiModalHashes prompt_updates: MultiModalPromptUpdates @@ -1137,23 +1150,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): ) -> Mapping[str, list[PlaceholderFeaturesInfo]]: return find_mm_placeholders(new_token_ids, mm_prompt_updates) - def _get_hf_mm_data( - self, - mm_items: MultiModalDataItems, - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - """Extract processor and passthrough data from multi-modal items.""" - processor_data = dict[str, object]() - passthrough_data = dict[str, object]() - - for items in mm_items.values(): - processor_data.update(items.get_processor_data()) - passthrough_data.update(items.get_passthrough_data()) - - return processor_data, passthrough_data - - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None: - """Get the text to pass to the HF processor alongside the multi-modal - data. + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None: + """Get the text to pass to the HF processor alongside the multi-modal data. By default, no text is passed. If the HF processor requires that text and multi-modal items correspond to each other, you should @@ -1162,29 +1160,53 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): """ return None - def _preprocess_hf_mm_data( + def _get_hf_mm_inputs( self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[Mapping[str, object], Mapping[str, object]]: - """Pre-process the multi-modal data and HF processor keyword arguments - before they are passed to the HF processor. + mm_items: MultiModalDataItems, + hf_kwargs: Mapping[str, object], + ) -> HFMultiModalInputs: + """Extract the multi-modal data and corresponding keyword arguments + for HF processing. - By default, both are returned as-is. If the HF processor expects the - multi-modal data under different keys than those provided by the - multi-modal items (e.g. `audio` instead of `audios`), or requires - additional keyword arguments (e.g. `sampling_rate`), you should - override this method. + If the HF processor expects the multi-modal data under different keys + than those provided by the multi-modal items, or requires additional + keyword arguments, you should override this method. """ - return mm_data, hf_processor_mm_kwargs + hf_data = dict[str, object]() + passthrough_data = dict[str, object]() + + for items in mm_items.values(): + if not items: + continue + + hf_data.update(items.get_processor_data()) + passthrough_data.update(items.get_passthrough_data()) + + if hf_data: + prompt_text = self._get_hf_mm_text(mm_items.get_all_counts()) + if prompt_text is not None: + hf_data = dict(text=prompt_text, **hf_data) + + # vLLM needs the untruncated sequence to keep placeholder + # tokens aligned. Note that the text inputs are just dummy + # text, not the original prompt. The original prompt is + # already tokenized by the renderer. + hf_kwargs = {"truncation": False, **hf_kwargs} + + # HF processors accept (images, text, videos, audio) + # so we need to remap vLLM's "audios" -> HF "audio" + if "audios" in hf_data: + hf_data["audio"] = hf_data.pop("audios") + + return HFMultiModalInputs(hf_data, hf_kwargs, passthrough_data) def _postprocess_hf_mm_data( self, - mm_data: Mapping[str, object], + mm_inputs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object], processed_data: BatchFeature, ) -> BatchFeature: - """Post-process the output of the HF processor. + """Post-process the combined processor and passthrough data. By default, the output is returned as-is. If you need to modify the output of the HF processor before it is converted into multi-modal @@ -1192,41 +1214,51 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): """ return processed_data + def _call_hf_processor( + self, + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], + ) -> BatchFeature: + """Call the HF processor on the normalized multi-modal inputs.""" + return self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_kwargs), + hf_data, + hf_kwargs, + ) + + def _finalize_hf_mm_data( + self, + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], + passthrough_data: Mapping[str, object], + processed_data: BatchFeature | None = None, + ) -> BatchFeature: + """Merge passthrough fields and apply model-specific postprocessing.""" + if processed_data is None: + from transformers.feature_extraction_utils import BatchFeature + + processed_data = BatchFeature() + + processed_data.update(passthrough_data) + return self._postprocess_hf_mm_data(hf_data, hf_kwargs, processed_data) + def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], + hf_kwargs: Mapping[str, object], ) -> BatchFeature: """Apply the HF processor on the multi-modal data.""" - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( + mm_items, hf_kwargs ) - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - if processor_data: - processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data( - processor_data, hf_processor_mm_kwargs - ) - - prompt_text = self._get_hf_processor_text(mm_items.get_all_counts()) - if prompt_text is not None: - processor_data = dict(text=prompt_text, **processor_data) - - processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - processor_data, - hf_processor_mm_kwargs, - ) - processed_data.update(passthrough_data) + if hf_data: + processed_data = self._call_hf_processor(hf_data, hf_kwargs) else: - from transformers.feature_extraction_utils import BatchFeature + processed_data = None - processed_data = BatchFeature(dict(passthrough_data)) - - return self._postprocess_hf_mm_data( - processor_data, - hf_processor_mm_kwargs, - processed_data, + return self._finalize_hf_mm_data( + hf_data, hf_kwargs, passthrough_data, processed_data ) def _postprocess_prompt(self, prompt: list[int]) -> list[int]: @@ -1261,6 +1293,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): mm_missing_data = {} for modality, idxs in mm_missing_idxs.items(): + if not idxs: + continue + missing_modality_data = [] for idx in idxs: data = mm_data_items[modality][idx] @@ -1271,6 +1306,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): ) else: missing_modality_data.append(data) + mm_missing_data[modality] = missing_modality_data mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False) @@ -1342,11 +1378,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> MultiModalProcessingInfo: + ) -> MultiModalProcessingResult: with timing_ctx.record("apply_hf_processor"): mm_processed_data = self._apply_hf_processor_main( mm_items=inputs.mm_data_items, - hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs, + hf_kwargs=inputs.hf_processor_mm_kwargs, ) mm_kwargs = MultiModalKwargsItems.from_hf_inputs( @@ -1369,26 +1405,28 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): mm_kwargs, ) - mm_info = MultiModalProcessingInfo( + return MultiModalProcessingResult( + prompt_ids=self._postprocess_prompt(inputs.prompt), kwargs=mm_kwargs, hashes=mm_hashes, prompt_updates=mm_prompt_updates, ) - return mm_info - def _cached_apply_hf_processor( self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> MultiModalProcessingInfo: + ) -> MultiModalProcessingResult: """Apply the HF processor on the full prompt text, caching the results and reusing cached results. """ cache = self.cache + has_passthrough_data = any( + len(items.get_passthrough_data()) > 0 + for items in inputs.mm_data_items.values() + ) - _, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items) - if cache is None or passthrough_data: + if cache is None or has_passthrough_data: return self._apply_hf_processor(inputs, timing_ctx) with timing_ctx.record("get_mm_hashes"): @@ -1410,7 +1448,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): with timing_ctx.record("apply_hf_processor"): mm_missing_processed_data = self._apply_hf_processor_main( mm_items=mm_missing_data_items, - hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs, + hf_kwargs=inputs.hf_processor_mm_kwargs, ) mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs( @@ -1435,14 +1473,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): mm_missing_prompt_updates=mm_missing_prompt_updates, ) - mm_info = MultiModalProcessingInfo( + return MultiModalProcessingResult( + prompt_ids=self._postprocess_prompt(inputs.prompt), kwargs=mm_kwargs, hashes=mm_hashes, prompt_updates=mm_prompt_updates, ) - return mm_info - def _apply_token_matches( self, prompt: list[int], @@ -1670,17 +1707,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): def _maybe_apply_prompt_updates( self, mm_items: MultiModalDataItems, - prompt_ids: list[int], - mm_kwargs: MultiModalKwargsOptionalItems, - mm_prompt_updates: MultiModalPromptUpdates, + mm_res: MultiModalProcessingResult, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: mm_item_counts = mm_items.get_all_counts() - self._validate_mm_kwargs(mm_kwargs, mm_item_counts) - self._validate_mm_updates(mm_prompt_updates, mm_item_counts) + self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts) + self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts) prompt_ids, mm_placeholders = self._apply_prompt_updates( - prompt_ids, - mm_prompt_updates, + mm_res.prompt_ids, + mm_res.prompt_updates, ) self._validate_mm_placeholders(mm_placeholders, mm_item_counts) @@ -1705,15 +1740,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): 4. Extract information about the placeholder tokens from the processed token IDs. """ - prompt_ids = self._postprocess_prompt(inputs.prompt) - mm_info = self._cached_apply_hf_processor(inputs, timing_ctx) + mm_res = self._cached_apply_hf_processor(inputs, timing_ctx) with timing_ctx.record("apply_prompt_updates"): prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates( mm_items=inputs.mm_data_items, - prompt_ids=prompt_ids, - mm_kwargs=mm_info.kwargs, - mm_prompt_updates=mm_info.prompt_updates, + mm_res=mm_res, ) mm_placeholder_ranges = { @@ -1723,8 +1755,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): return mm_input( prompt_token_ids=prompt_ids, - mm_kwargs=mm_info.kwargs, - mm_hashes=mm_info.hashes, + mm_kwargs=mm_res.kwargs, + mm_hashes=mm_res.hashes, mm_placeholders=mm_placeholder_ranges, )