[MM] Simplify _apply_hf_processor_main (#53275)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

docs/contributing/model/multimodal.md

@@ -153,11 +153,13 @@ def get_supported_mm_limits(self) -> Mapping[str, int | None]:
 
 ## 3. Specify dummy inputs
 
-Then, inherit [BaseDummyInputsBuilder][vllm.multimodal.processing.BaseDummyInputsBuilder] to construct dummy inputs for
-HF processing. The processed outputs are also used for memory profiling.
+Then, inherit [BaseDummyInputsBuilder][vllm.multimodal.processing.BaseDummyInputsBuilder] to construct the dummy inputs
+that are used for memory profiling.
 
 Override the abstract methods [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text] and [get_dummy_mm_data][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_mm_data] to construct dummy inputs. These dummy inputs should result in the worst-case memory usage of the model so that vLLM can reserve the correct amount of memory for it.
 
+Besides profiling, the dummy text is passed to the HF processor by models whose HF processor requires text corresponding to the multi-modal items (see [Multi-modal fields](#multi-modal-fields)). If the HF processor can process multi-modal data without any text, the dummy text is not used during HF processing.
+
 Assuming that the memory usage increases with the number of tokens, the dummy inputs can be constructed to maximize the number of output embeddings, which is the same number as placeholder feature tokens.
 
 === "Basic example: LLaVA"
@@ -446,37 +448,64 @@ return a schema of the tensors outputted by the HF processor that are related to
     like in LLaVA, each image's features must be independent of the others (which
     is also required for prefix caching to work correctly). So, we un-pad each image
     back to its own size by overriding
-    [BaseMultiModalProcessor._call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor]:
+    [BaseMultiModalProcessor._postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data]
+    to post-process the outputs of the HF processor:
 
     ??? code
 
         ```python
-        def _call_hf_processor(
-            self,
-            prompt: str,
-            mm_data: Mapping[str, object],
-            mm_kwargs: Mapping[str, object],
-        ) -> BatchFeature:
-            processed_outputs = super()._call_hf_processor(
-                prompt=prompt,
-                mm_data=mm_data,
-                mm_kwargs=mm_kwargs,
-            )
+        def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+            # Mistral3Processor requires text corresponding to the images
+            return self.dummy_inputs.get_dummy_text(mm_counts)
 
-            pixel_values = processed_outputs.get("pixel_values")
+        def _postprocess_hf_mm_data(
+            self,
+            mm_data: Mapping[str, object],
+            hf_processor_mm_kwargs: Mapping[str, object],
+            processed_data: BatchFeature,
+        ) -> BatchFeature:
+            if not mm_data:
+                return processed_data
+
+            pixel_values = processed_data.get("pixel_values")
             if pixel_values is not None:
                 # Avoid padding since we need the output for each image to be
                 # independent of other images for the cache to work correctly
-                image_sizes = processed_outputs["image_sizes"]
+                image_sizes = processed_data["image_sizes"]
                 assert len(pixel_values) == len(image_sizes)
 
-                processed_outputs["pixel_values"] = [
+                processed_data["pixel_values"] = [
                     p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
                 ]
 
-            return processed_outputs
+            return processed_data
         ```
 
+    The default implementation of
+    [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]
+    calls the HF processor on the multi-modal data without passing any text.
+    If the HF processor instead requires text corresponding to the multi-modal items,
+    you should override
+    [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text]
+    to return the dummy text from
+    [BaseDummyInputsBuilder.get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text]
+    like in the example above. If the HF processor expects the multi-modal data
+    under different keys than those provided by the multi-modal items
+    (e.g. `audio` instead of `audios`), or requires additional keyword arguments
+    (e.g. `sampling_rate`), you should override
+    [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data].
+    If you need to modify the output of the HF processor,
+    you should override
+    [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data].
+    If the HF processor applies additional transformations to the prompt tokens
+    that are not reflected in its multi-modal outputs (e.g. prepending a BOS token),
+    you should override
+    [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt]
+    to replicate them.
+    For even more control over how the HF processor is called, you can override
+    [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]
+    directly.
+
     Since `pixel_values` is now a list with one tensor per image, we can override
     [_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows:
 
@@ -680,7 +709,7 @@ Examples:
 
 ### Handling prompt updates unrelated to multi-modal data
 
-[_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates] assumes that each application of prompt update corresponds to one multi-modal item. If the HF processor performs additional processing regardless of how many multi-modal items there are, you should override [_apply_hf_processor_tokens_only][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_tokens_only] so that the processed token inputs are consistent with the result of applying the HF processor on text inputs. This is because token inputs bypass the HF processor according to [our design](../../design/mm_processing.md).
+[_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates] assumes that each application of prompt update corresponds to one multi-modal item. If the HF processor performs additional processing on the prompt regardless of how many multi-modal items there are, you should override [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt] so that the prompt token IDs are consistent with the result of applying the HF processor on text inputs. This is because token inputs bypass the HF processor according to [our design](../../design/mm_processing.md).
 
 Examples:
 
@@ -690,7 +719,7 @@ Examples:
 
 ### Custom HF processor
 
-Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and pass it to [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor].
+Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. It is then applied to the multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor] inside [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main].
 
 Examples:
 

docs/design/mm_processing.md

@@ -2,7 +2,15 @@
 
 To enable various optimizations in vLLM such as [chunked prefill](../configuration/optimization.md#chunked-prefill) and [prefix caching](../features/automatic_prefix_caching.md), we use [BaseMultiModalProcessor][vllm.multimodal.processing.BaseMultiModalProcessor] to provide the correspondence between placeholder feature tokens (e.g. `<image>`) and multi-modal inputs (e.g. the raw input image) based on the outputs of HF processor.
 
-Here are the main features of [BaseMultiModalProcessor][vllm.multimodal.processing.BaseMultiModalProcessor]:
+In vLLM's rendering pipeline (see [BaseRenderer][vllm.renderers.base.BaseRenderer]), tokenization is performed as a separate step before multi-modal processing. Therefore, `BaseMultiModalProcessor` needs to recreate the output of calling HF processor end-to-end, while not being able to see the original text. This is achieved through **Dummy Input Text** and **Prompt Update Detection**.
+
+## Dummy Input Text
+
+Since Transformers 5.10, `ProcessorMixin` now allows multi-modal inputs to be passed by themselves. However, certain subclasses (such as `ChameleonProcessor`) and older out-of-tree implementations may still define their own `__call__` method that assumes the presence of text with corresponding placeholder tokens. This causes a problem as we don't have the original text anymore to pass to these processors.
+
+To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text] returns so that [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] passes it to the HF processor together with the multi-modal inputs to obtain the processed multi-modal data.
+
+Similarly, since the multi-modal data extracted by vLLM may not match what a specific HF processor expects, [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] allows each model to adapt the inputs via [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data] (e.g. renaming keys like `audios` to `audio` or injecting extra keyword arguments such as `sampling_rate`) and the outputs via [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data], without having to reimplement the entire method.
 
 ## Prompt Update Detection
 
@@ -13,46 +21,9 @@ One of the main responsibilities of HF processor is to update the prompt with pl
 
 The information about which tokens have been updated is key to finding the correspondence between placeholder feature tokens and multi-modal inputs.
 
-In vLLM, this information is specified using [PromptUpdate][vllm.multimodal.processing.PromptUpdate] in [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates]. We can automatically detect whether HF has updated the prompt by checking the existence of the updated tokens.
+Since we call HF processor without the input text, we have to perform this update by ourselves. In vLLM, we represent the necessary information using [PromptUpdate][vllm.multimodal.processing.PromptUpdate] in [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates], and apply them via [_apply_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._apply_prompt_updates].
 
-## Tokenized Prompt Inputs
-
-To enable tokenization in a separate process, we support passing input token IDs alongside multi-modal data.
-
-### The problem
-
-Consider that HF processors follow these main steps:
-
-1. Tokenize the text
-2. Process multi-modal inputs
-3. Perform prompt updates
-
-And we require that:
-
-- For text + multi-modal inputs, apply all steps 1--3.
-- For tokenized + multi-modal inputs, apply only steps 2--3.
-
-How can we achieve this without rewriting HF processors? We can try to call the HF processor several times on different inputs:
-
-- For text + multi-modal inputs, simply call the HF processor directly.
-- For tokenized + multi-modal inputs, call the processor only on the multi-modal inputs.
-
-While HF processors support text + multi-modal inputs natively, this is not so for tokenized + multi-modal inputs: an error is thrown if the number of input placeholder tokens do not correspond to the number of multi-modal inputs.
-
-Moreover, since the tokenized text has not passed through the HF processor, we have to apply Step 3 by ourselves to keep the output tokens and multi-modal data consistent with each other.
-
-### Dummy text
-
-We work around the first issue by requiring each model to define how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text]. This lets us generate dummy text corresponding to the multi-modal inputs and input them together to obtain the processed multi-modal data.
-
-### Automatic prompt updating
-
-We address the second issue by implementing model-agnostic code in
-[_apply_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._apply_prompt_updates] to automatically update the prompt with feature placeholder tokens based on the specification outputted by [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates].
-
-### Summary
-
-With the help of dummy text and automatic prompt updating, our multi-modal processor can finally accept both text and token prompts with multi-modal data. The detailed logic is shown in [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main].
+Some HF processors additionally transform the prompt itself regardless of the multi-modal inputs (such as `ChameleonProcessor` appending a sep token for chat mode). Since the prompt tokens likewise bypass the HF processor, such transformations are replicated via [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt] before the prompt updates are located or applied.
 
 ## Processor Output Caching
 
@@ -60,8 +31,6 @@ Some HF processors, such as the one for Qwen2-VL, are [very slow](https://github
 
 When new data is passed in, we first check which items are in the cache, and which ones are missing. The missing items are passed into the HF processor in a single batch and cached, before being merged with the existing items in the cache.
 
-Since we only process the missing multi-modal data items, the number of input placeholder tokens no longer corresponds to the number of the multi-modal inputs, so they can't be passed alongside the text prompt to HF processor. Therefore, we process the text and multi-modal inputs separately, using [dummy text](#dummy-text) to avoid HF errors. Since this skips HF's prompt updating code, we apply [automatic prompt updating](#automatic-prompt-updating) afterwards to keep the output tokens and multi-modal data consistent with each other.
-
 ## Speeding Up Multi‑Modal Data Processing
 
 ### Fused Normalisation on the Device

requirements/common.txt

@@ -7,7 +7,7 @@ requests >= 2.26.0
 tqdm
 blake3
 py-cpuinfo
-transformers >= 5.5.3
+transformers >= 5.10.4
 huggingface_hub >= 1.28.0
 tokenizers >= 0.21.1  # Required for fast incremental detokenization.
 safetensors >= 0.6.2  # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611

tests/models/multimodal/processing/test_audio_in_video.py

@@ -136,7 +136,7 @@ def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
     # no audio track" case. (The audio limit above is 1, not 0, so that
     # `mm_counts["audio"]` is still declared for this request; a 0 limit
     # would make vLLM omit the "audio" key from `mm_counts` entirely, which
-    # trips an unrelated assertion in `_apply_hf_processor_mm_only`.)
+    # trips an unrelated assertion in `_apply_hf_processor_main`.)
     mm_data = {"video": [video]}
 
     with pytest.raises(ValueError, match="doesn't have audio track"):

tests/models/multimodal/processing/test_audioflamingo3.py

@@ -105,9 +105,9 @@ def test_audio_chunk_counting(mock_ctx):
     audio_2 = np.zeros(75 * sr)
 
     mm_data = {"audio": [audio_1, audio_2]}
-    prompt = "<|user|>Listen.<|end|>"
 
-    processed = processor._call_hf_processor(prompt, mm_data, {})
+    mm_items = processor.info.parse_mm_data(mm_data, validate=False)
+    processed = processor._apply_hf_processor_main(mm_items, {})
 
     chunk_counts = processed["chunk_counts"]
 

tests/models/multimodal/processing/test_common.py

@@ -385,17 +385,19 @@ def test_processing_correctness(
     num_batches: int,
     simplify_rate: float,
 ):
-    if model_id == "google/gemma-3n-E2B-it":
-        pytest.skip("Fix later")
-    if model_id == "OpenGVLab/InternVL2-2B":
-        pytest.skip("Fix later")
     if model_id == "openvla/openvla-7b":
         pytest.skip(
             "OpenVLA uses a custom vLLM processor because its HF remote "
             "processor is incompatible with current Transformers."
         )
     if model_id == "jinaai/jina-reranker-m0":
-        pytest.skip("Fix later")
+        pytest.skip(
+            "JinaVL reverses the multi-modal item order inside "
+            "_apply_hf_processor_main to match its document-before-query "
+            "score template, which breaks the positional correspondence "
+            "between multi-modal kwargs and hashes assumed by the "
+            "processing cache. Needs a fix that keeps both consistent."
+        )
     if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
         pytest.skip(
             "Voxtral Realtime doesn't make use of any place-holder "
@@ -403,22 +405,12 @@ def test_processing_correctness(
             "correctness test as is. Let's revisit adapting this "
             "test once more realtime models exist."
         )
-    if model_id == "CohereLabs/cohere-transcribe-03-2026":
-        pytest.skip("Fix later")
     if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"):
         pytest.skip(
             "MOSS-Audio uses a custom processor that dynamically expands "
             "audio placeholders from processed audio lengths. Its vLLM "
             "processor paths are covered by test_moss_audio.py."
         )
-    # TODO: Remove when transformers 5.15.0 is released, which contains
-    # https://github.com/huggingface/transformers/pull/47483.
-    if model_id == "microsoft/VibeVoice-ASR-HF":
-        pytest.skip(
-            "VibeVoice ASR requires audio as a positional argument and hence "
-            "cannot pass the processing correctness test as is. Its generation "
-            "is covered by test_transformers_audio.py."
-        )
     if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
         pytest.skip(
             "LLaVA-OneVision-2 video processing routes frames through custom "

tests/models/multimodal/processing/test_llava_onevision2.py

@@ -202,7 +202,7 @@ def test_validate_relative_bare_path_blocked():
 # ---------------------------------------------------------------------------
 # Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps)
 #
-# Non-codec videos reach _call_hf_processor as a ``(frames_ndarray, metadata)``
+# Non-codec videos reach _apply_hf_processor_main as a ``(frames_ndarray, metadata)``
 # tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real
 # ``video_url`` inputs, or by the dummy-inputs builder during profiling
 # (``video_needs_metadata=True``). The helper materialises PIL frames and

tests/models/multimodal/processing/test_qwen3_vl.py

@@ -103,7 +103,7 @@ def test_processor_multi_video(
     """Verify that multi-video processing produces correct placeholders.
 
     This exercises the token-level replacement path in
-    ``_call_hf_processor`` which avoids the quadratic text-level
+    ``_apply_hf_processor_main`` which avoids the quadratic text-level
     prompt expansion.
     """
     ctx = build_model_context(
@@ -153,7 +153,7 @@ def test_processor_multi_video_list_kwargs(
     ``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
     crash.
 
-    Before the fix, ``_call_hf_processor`` copied the whole kwargs to every
+    Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
     video without slicing, so ``_get_video_second_idx`` received the list
     where a scalar was expected and raised ``TypeError``.
     """

vllm/model_executor/models/audioflamingo3.py

@@ -377,35 +377,41 @@ class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):
 class AudioFlamingo3MultiModalProcessor(
     BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo]
 ):
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, Any],
-    ) -> BatchFeature:
-        processor_mm_data = dict(mm_data)
-        audios = processor_mm_data.pop("audios", None)
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        mm_data = dict(mm_data)
+        audios = mm_data.pop("audios", None)
         if audios is not None:
-            processor_mm_data["audio"] = audios
+            mm_data["audio"] = audios
 
-        outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=processor_mm_data,
-            mm_kwargs=mm_kwargs,
-        )
+        return mm_data, hf_processor_mm_kwargs
 
-        if "input_features_mask" in outputs:
-            outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if "input_features_mask" in processed_data:
+            processed_data["feature_attention_mask"] = processed_data.pop(
+                "input_features_mask"
+            )
 
-        audio_data = processor_mm_data.get("audio")
+        audio_data = mm_data.get("audio")
         if audio_data is None:
-            return outputs
+            return processed_data
 
         audio_list = audio_data if isinstance(audio_data, list) else [audio_data]
         if len(audio_list) == 0:
-            return outputs
+            return processed_data
 
-        processor = self.info.get_hf_processor(**mm_kwargs)
+        processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
         feature_extractor = processor.feature_extractor
         sampling_rate = feature_extractor.sampling_rate
         chunk_length = feature_extractor.chunk_length
@@ -423,8 +429,9 @@ class AudioFlamingo3MultiModalProcessor(
                 n_win = max_windows
             chunk_counts.append(n_win)
 
-        outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
-        return outputs
+        processed_data["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/blip2.py

@@ -467,24 +467,6 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
 
 
 class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        if not mm_data:
-            # HF processor always adds placeholders even when there's no image
-            tokenizer = self.info.get_tokenizer()
-            prompt_ids = tokenizer.encode(prompt)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
-
-        return super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
-
     def _get_mm_fields_config(
         self,
         hf_inputs: BatchFeature,

vllm/model_executor/models/chameleon.py

@@ -135,34 +135,16 @@ class ChameleonDummyInputsBuilder(BaseDummyInputsBuilder[ChameleonProcessingInfo
 
 
 class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingInfo]):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        if not mm_data:
-            prompt_ids = self.info.get_tokenizer().encode(prompt)
-            prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        # ChameleonProcessor requires text corresponding to the images
+        return self.dummy_inputs.get_dummy_text(mm_counts)
 
-        return super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
-
-    def _apply_hf_processor_tokens_only(
-        self,
-        prompt_tokens: list[int],
-    ) -> list[int]:
+    def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
         # HF processor adds sep token for chat mode
         tokenizer = self.info.get_tokenizer()
-        vocab = tokenizer.get_vocab()
+        sep_token_id = tokenizer.get_vocab()[tokenizer.sep_token]  # type: ignore
 
-        sep_token_id = vocab[tokenizer.sep_token]  # type: ignore
-
-        return prompt_tokens + [sep_token_id]
+        return [*prompt, sep_token_id]
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/cheers.py

@@ -16,7 +16,6 @@ import torch
 import torch.nn as nn
 import torch.nn.functional as F
 from einops import rearrange
-from transformers import BatchFeature
 
 from vllm.config import VllmConfig
 from vllm.config.multimodal import BaseDummyOptions
@@ -492,14 +491,6 @@ class CheersDummyInputsBuilder(BaseDummyInputsBuilder[CheersProcessingInfo]):
 class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):
     """Multimodal processor for Cheers model."""
 
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
-
     def _get_prompt_updates(
         self,
         mm_items: MultiModalDataItems,

vllm/model_executor/models/cohere2_vision.py

@@ -222,24 +222,23 @@ class Cohere2VisionDummyInputsBuilder(
 class Cohere2VisionMultiModalProcessor(
     BaseMultiModalProcessor[Cohere2VisionProcessingInfo]
 ):
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _postprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
     ) -> BatchFeature:
-        processed_outputs = super()._call_hf_processor(
-            prompt,
-            mm_data,
-            mm_kwargs,
-        )
+        if not mm_data:
+            return processed_data
 
-        # Ensure num_patches is available for proper tensor splitting
         if (
-            "num_patches" not in processed_outputs
+            "num_patches" not in processed_data
             and (images := mm_data.get("images")) is not None
         ):
-            hf_processor = self.info.get_hf_processor(**mm_kwargs)
+            hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
 
             # Fallback calculation if HF processor didn't provide num_patches
             mm_items = self.info.parse_mm_data({"image": images}, validate=False)
@@ -250,13 +249,13 @@ class Cohere2VisionMultiModalProcessor(
                     image_width=parsed_images.get_image_size(i).width,
                     image_height=parsed_images.get_image_size(i).height,
                     processor=hf_processor,
-                    mm_kwargs=mm_kwargs,
+                    mm_kwargs=hf_processor_mm_kwargs,
                 )
                 for i in range(len(parsed_images))
             ]
-            processed_outputs["num_patches"] = torch.tensor(num_patches)
+            processed_data["num_patches"] = torch.tensor(num_patches)
 
-        return processed_outputs
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/cohere_asr.py

@@ -8,7 +8,7 @@ from typing import Any, ClassVar
 import torch
 import torch.nn.functional as F
 from torch import nn
-from transformers import PretrainedConfig
+from transformers import BatchFeature, PretrainedConfig
 
 from vllm.compilation.decorators import support_torch_compile
 from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
@@ -45,9 +45,12 @@ from vllm.multimodal.processing import (
     BaseDummyInputsBuilder,
     BaseProcessingInfo,
     EncDecMultiModalProcessor,
+    ProcessorInputs,
     PromptReplacement,
     PromptUpdate,
+    TimingContext,
 )
+from vllm.multimodal.processing.processor import MultiModalProcessingInfo
 from vllm.renderers import TokenizeParams
 from vllm.tokenizers import cached_tokenizer_from_config
 from vllm.transformers_utils.processors.cohere_asr import (
@@ -1941,27 +1944,51 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
     ) -> list[int]:
         return [0]
 
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ):
-        if mm_data:
-            feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
-            mm_data = dict(audio=dict(mm_data).pop("audios"))
-            mm_kwargs = dict(
-                **mm_kwargs,
-                sampling_rate=feature_extractor.sampling_rate,
-            )
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
+
+        mm_data = dict(mm_data)
+        mm_data["audio"] = mm_data.pop("audios")
+
+        hf_processor_mm_kwargs = dict(
+            **hf_processor_mm_kwargs,
+            sampling_rate=feature_extractor.sampling_rate,
         )
-        if "labels" in processed_outputs:
-            processed_outputs["input_ids"] = processed_outputs.pop("labels")
-        return processed_outputs
+
+        return mm_data, hf_processor_mm_kwargs
+
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if "labels" in processed_data:
+            processed_data["input_ids"] = processed_data.pop("labels")
+
+        return processed_data
+
+    def _cached_apply_hf_processor(
+        self,
+        inputs: ProcessorInputs,
+        timing_ctx: TimingContext,
+    ) -> MultiModalProcessingInfo:
+        # Dithering injects noise into the extracted features, so the
+        # feature extractor is not a pure function of its input. Since the
+        # processing cache assumes that processor outputs are invariant
+        # across calls, bypass the cache when dithering is active.
+        preproc = self.info.get_hf_config().preprocessor
+        if preproc.get("dither", 1e-05) > 0:
+            return self._apply_hf_processor(inputs, timing_ctx)
+
+        return super()._cached_apply_hf_processor(inputs, timing_ctx)
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/colmodernvbert.py

@@ -154,16 +154,25 @@ class ColModernVBertDummyInputsBuilder(
 class ColModernVBertMultiModalProcessor(
     BaseMultiModalProcessor[ColModernVBertProcessingInfo],
 ):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
         tokenizer = self.info.get_tokenizer()
         assert isinstance(tokenizer, HfTokenizer)
         text_encoding = tokenizer(
-            prompt,
+            prompt_text,
             return_tensors="pt",
         )
         result = BatchFeature(data=dict(text_encoding))
@@ -183,7 +192,9 @@ class ColModernVBertMultiModalProcessor(
             )
             result.update(image_outputs)
 
-        return result
+        processed_data = result
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/colpali.py

@@ -28,6 +28,7 @@ from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
 from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.parse import MultiModalDataItems
 
 from .interfaces import SupportsLateInteraction
 from .interfaces_base import default_pooling_type
@@ -59,24 +60,31 @@ class ColPaliProcessingInfo(PaliGemmaProcessingInfo):
 class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
     """Multimodal processor for ColPali."""
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        if mm_data:
-            # The ColPali tokenizer_config.json ships with a small default
-            # max_length (50) that truncates the 1024 image tokens inserted
-            # by PaliGemmaProcessor, causing a token-count mismatch.
-            # vLLM enforces its own max_model_len, so we disable HF
-            # truncation to keep all image + text tokens intact.
-            return self.info.ctx.call_hf_processor(
-                self.info.get_hf_processor(**mm_kwargs),
-                dict(text=prompt, **mm_data),
-                dict(**mm_kwargs, truncation=False),
-            )
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        # The ColPali tokenizer_config.json ships with a small default
+        # max_length (50) that truncates the 1024 image tokens inserted
+        # by PaliGemmaProcessor, causing a token-count mismatch.
+        # vLLM enforces its own max_model_len, so we disable HF
+        # truncation to keep all image + text tokens intact.
+        processed_data = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**hf_processor_mm_kwargs),
+            mm_data,
+            dict(**hf_processor_mm_kwargs, truncation=False),
+        )
+        processed_data.update(passthrough_data)
+        return processed_data
 
 
 @default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")

vllm/model_executor/models/deepseek_ocr.py

@@ -287,27 +287,34 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing
 class DeepseekOCRMultiModalProcessor(
     BaseMultiModalProcessor[DeepseekOCRProcessingInfo]
 ):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
         if mm_data:
-            processed_outputs = self.info.ctx.call_hf_processor(
-                self.info.get_hf_processor(**mm_kwargs),
-                dict(prompt=prompt, **mm_data),
-                mm_kwargs,
+            processed_data = self.info.ctx.call_hf_processor(
+                self.info.get_hf_processor(**hf_processor_mm_kwargs),
+                dict(prompt=prompt_text, **mm_data),
+                hf_processor_mm_kwargs,
             )
 
         else:
             tokenizer = self.info.get_tokenizer()
             assert isinstance(tokenizer, HfTokenizer)
-            processed_outputs = tokenizer(
-                prompt, add_special_tokens=True, return_tensors="pt"
+            processed_data = tokenizer(
+                prompt_text, add_special_tokens=True, return_tensors="pt"
             )
 
-        return processed_outputs
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/deepseek_ocr2.py

@@ -158,27 +158,35 @@ class DeepseekOCR2DummyInputsBuilder(
 class DeepseekOCR2MultiModalProcessor(
     BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo]
 ):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
         if mm_data:
-            processed_outputs = self.info.ctx.call_hf_processor(
-                self.info.get_hf_processor(**mm_kwargs),
-                dict(prompt=prompt, **mm_data),
-                mm_kwargs,
+            processed_data = self.info.ctx.call_hf_processor(
+                self.info.get_hf_processor(**hf_processor_mm_kwargs),
+                dict(prompt=prompt_text, **mm_data),
+                hf_processor_mm_kwargs,
             )
 
         else:
             tokenizer = self.info.get_tokenizer()
             assert isinstance(tokenizer, HfTokenizer)
-            processed_outputs = tokenizer(
-                prompt, add_special_tokens=True, return_tensors="pt"
+            processed_data = tokenizer(
+                prompt_text, add_special_tokens=True, return_tensors="pt"
             )
 
-        return processed_outputs
+        processed_data.update(passthrough_data)
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/deepseek_vl2.py

@@ -43,7 +43,6 @@ from vllm.multimodal.processing.processor import (
 )
 from vllm.sequence import IntermediateTensors
 from vllm.tokenizers import cached_tokenizer_from_config
-from vllm.tokenizers.hf import HfTokenizer
 from vllm.transformers_utils.configs.deepseek_vl2 import (
     DeepseekVLV2Config,
     MlpProjectorConfig,
@@ -236,28 +235,23 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
 class DeepseekVL2MultiModalProcessor(
     BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]
 ):
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _postprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
     ) -> BatchFeature:
         if not mm_data:
-            tokenizer = self.info.get_tokenizer()
-            assert isinstance(tokenizer, HfTokenizer)
-            return tokenizer(prompt, add_special_tokens=True, return_tensors="pt")
+            return processed_data
 
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
+        processed_data["num_patches"] = (
+            processed_data["images_spatial_crop"].prod(-1) + 1
         )
 
-        processed_outputs["num_patches"] = (
-            processed_outputs["images_spatial_crop"].prod(-1) + 1
-        )
-
-        return processed_outputs
+        return processed_data
 
     def _get_mm_fields_config(
         self,
@@ -315,7 +309,7 @@ class DeepseekVL2MultiModalProcessor(
         self,
         inputs: ProcessorInputs,
         timing_ctx: TimingContext,
-    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
+    ) -> MultiModalProcessingInfo:
         # The processor logic is different for len(images) <= 2 vs > 2
         # Since the processing cache assumes that the processor output is
         # invariant of how many images are passed per prompt, we only

vllm/model_executor/models/ernie45_vl.py

@@ -1110,21 +1110,20 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
         pixel_values = pixel_values.to(hf_config.dtype)
         return pixel_values
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        # when the prompt is not empty but the multimodal data is empty,
-        # directly invoke the tokenizer.
-        if "images" not in mm_data and "videos" not in mm_data and prompt != "":
-            tokenizer = self.info.get_tokenizer()
-            prompt_ids = tokenizer.encode(prompt)
-            tokenizer_output = BatchFeature(
-                dict(input_ids=[prompt_ids]), tensor_type="pt"
-            )
-            return tokenizer_output
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if "images" not in mm_data and "videos" not in mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
 
         if "images" not in mm_data:
             mm_data["images"] = []
@@ -1132,7 +1131,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
             mm_data["videos"] = []
 
         # Check if HF processor supports video metadata
-        hf_processor = self.info.get_hf_processor(**mm_kwargs)
+        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
         supports_video_metadata = getattr(
             hf_processor, "supports_video_metadata", False
         )
@@ -1149,8 +1148,10 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
 
         processor_output = self.info.ctx.call_hf_processor(
             hf_processor,
-            dict(text=[prompt], images=mm_data["images"], videos=mm_data["videos"]),
-            mm_kwargs,
+            dict(
+                text=[prompt_text], images=mm_data["images"], videos=mm_data["videos"]
+            ),
+            hf_processor_mm_kwargs,
         )
 
         # Divide the processor_output into two modalities: image and video.
@@ -1158,7 +1159,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
             pixel_values = processor_output["images"]
             if pixel_values is not None:
                 processor_output["images"] = self._pixel_values_norm(
-                    pixel_values, mm_kwargs
+                    pixel_values, hf_processor_mm_kwargs
                 )
             for key in list(processor_output.keys()):
                 if processor_output[key] is None:
@@ -1184,7 +1185,9 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
                     ]
                     del processor_output["images"]
 
-        return processor_output
+        processed_data = processor_output
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_prompt_updates(
         self,

vllm/model_executor/models/fireredasr2.py

@@ -228,27 +228,36 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing
 class FireRedASR2MultiModalProcessor(
     BaseMultiModalProcessor[FireRedASR2ProcessingInfo]
 ):
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        if mm_data:
-            feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
-            mm_data = dict(audio=mm_data.pop("audios"))
-            mm_kwargs = dict(
-                **mm_kwargs,
-                sampling_rate=feature_extractor.sampling_rate,
-            )
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
+
+        mm_data = dict(mm_data)
+        mm_data["audio"] = mm_data.pop("audios")
+
+        hf_processor_mm_kwargs = dict(
+            **hf_processor_mm_kwargs,
+            sampling_rate=feature_extractor.sampling_rate,
         )
-        if "labels" in processed_outputs:
-            processed_outputs["input_ids"] = processed_outputs.pop("labels")
-        return processed_outputs
+
+        return mm_data, hf_processor_mm_kwargs
+
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if "labels" in processed_data:
+            processed_data["input_ids"] = processed_data.pop("labels")
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/fireredlid.py

@@ -451,27 +451,36 @@ class FireRedLIDMultiModalProcessor(
         # Dummy encoder prompt for profiling (encoder only processes audio).
         return [0]
 
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        if mm_data:
-            feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
-            mm_data = dict(audio=mm_data.pop("audios"))
-            mm_kwargs = dict(
-                **mm_kwargs,
-                sampling_rate=feature_extractor.sampling_rate,
-            )
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
+
+        mm_data = dict(mm_data)
+        mm_data["audio"] = mm_data.pop("audios")
+
+        hf_processor_mm_kwargs = dict(
+            **hf_processor_mm_kwargs,
+            sampling_rate=feature_extractor.sampling_rate,
         )
-        if "labels" in processed_outputs:
-            processed_outputs["input_ids"] = processed_outputs.pop("labels")
-        return processed_outputs
+
+        return mm_data, hf_processor_mm_kwargs
+
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if "labels" in processed_data:
+            processed_data["input_ids"] = processed_data.pop("labels")
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/funasr.py

@@ -44,9 +44,12 @@ from vllm.multimodal.processing import (
     BaseDummyInputsBuilder,
     BaseMultiModalProcessor,
     BaseProcessingInfo,
+    ProcessorInputs,
     PromptReplacement,
     PromptUpdate,
+    TimingContext,
 )
+from vllm.multimodal.processing.processor import MultiModalProcessingInfo
 from vllm.transformers_utils.processor import cached_processor_from_config
 from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor
 from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -747,27 +750,50 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
 
 
 class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        if mm_data:
-            feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
-            mm_data = dict(audio=mm_data.pop("audios"))
-            mm_kwargs = dict(
-                **mm_kwargs,
-                sampling_rate=feature_extractor.sampling_rate,
-            )
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
+
+        mm_data = dict(mm_data)
+        mm_data["audio"] = mm_data.pop("audios")
+
+        hf_processor_mm_kwargs = dict(
+            **hf_processor_mm_kwargs,
+            sampling_rate=feature_extractor.sampling_rate,
         )
-        if "labels" in processed_outputs:
-            processed_outputs["input_ids"] = processed_outputs.pop("labels")
-        return processed_outputs
+
+        return mm_data, hf_processor_mm_kwargs
+
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if "labels" in processed_data:
+            processed_data["input_ids"] = processed_data.pop("labels")
+
+        return processed_data
+
+    def _cached_apply_hf_processor(
+        self,
+        inputs: ProcessorInputs,
+        timing_ctx: TimingContext,
+    ) -> MultiModalProcessingInfo:
+        # Dithering injects noise into the extracted features, so the
+        # feature extractor is not a pure function of its input. Since the
+        # processing cache assumes that processor outputs are invariant
+        # across calls, bypass the cache when dithering is active.
+        if self.info.get_feature_extractor().dither > 0:
+            return self._apply_hf_processor(inputs, timing_ctx)
+
+        return super()._cached_apply_hf_processor(inputs, timing_ctx)
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/funaudiochat.py

@@ -612,20 +612,31 @@ class FunAudioChatDummyInputsBuilder(
 class FunAudioChatMultiModalProcessor(
     BaseMultiModalProcessor[FunAudioChatProcessingInfo]
 ):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
         tokenizer = self.info.get_tokenizer()
-        input_ids = torch.tensor([tokenizer.encode(prompt)])
+        input_ids = torch.tensor([tokenizer.encode(prompt_text)])
 
         audios = mm_data.get("audios", [])
         if not audios:
-            return BatchFeature({"input_ids": input_ids})
+            processed_data = BatchFeature({"input_ids": input_ids})
+            processed_data.update(passthrough_data)
+            return processed_data
 
-        feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
+        feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
         sr = int(feature_extractor.sampling_rate)
         min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400)
 
@@ -676,7 +687,9 @@ class FunAudioChatMultiModalProcessor(
             "feature_exist_mask": torch.ones((len(wavs),), dtype=torch.bool),
         }
 
-        return BatchFeature({"input_ids": input_ids, **mm_inputs})
+        processed_data = BatchFeature({"input_ids": input_ids, **mm_inputs})
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/gemma3_mm.py

@@ -261,39 +261,38 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
 
 
 class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _postprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
     ) -> BatchFeature:
-        processed_outputs = super()._call_hf_processor(
-            prompt,
-            mm_data,
-            mm_kwargs,
-        )
+        if not mm_data:
+            return processed_data
 
-        # HF processor pops the `num_crops` kwarg, which is needed by vLLM
         if (images := mm_data.get("images")) is not None:
             mm_items = self.info.parse_mm_data({"image": images}, validate=False)
             parsed_images = mm_items.get_items("image", ImageProcessorItems)
             image_sizes = [
                 parsed_images.get_image_size(i) for i in range(len(parsed_images))
             ]
-            hf_processor = self.info.get_hf_processor(**mm_kwargs)
+            hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
 
             num_crops = [
                 self.info.get_num_crops(
                     image_width=size.width,
                     image_height=size.height,
                     processor=hf_processor,
-                    mm_kwargs=mm_kwargs,
+                    mm_kwargs=hf_processor_mm_kwargs,
                 )
                 for size in image_sizes
             ]
-            processed_outputs["num_patches"] = torch.tensor(num_crops) + 1
+            processed_data["num_patches"] = torch.tensor(num_crops) + 1
 
-        return processed_outputs
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/gemma3n_mm.py

@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import inspect
 from collections.abc import Iterable, Mapping, Sequence
 from typing import Annotated, Any, Literal
 
@@ -257,40 +258,69 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
 
 
 class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]):
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        # HF Transformers audio processor no longer accepts `audios` key.
-        # We pop `audios` and replace it with `audio` key to suppress
-        # the warning.
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        mm_data = dict(mm_data)
         if "audios" in mm_data:
             mm_data["audio"] = mm_data.pop("audios")
-        processed_outputs = super()._call_hf_processor(
-            prompt,
-            mm_data,
-            mm_kwargs,
-        )
 
-        if "input_features" in processed_outputs:
-            # Padding enables audio_tower to run in batched mode
-            processed_outputs["input_features_padded"] = processed_outputs[
-                "input_features"
+        return mm_data, hf_processor_mm_kwargs
+
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if "input_features" in processed_data:
+            # The feature extractor pads every clip to the longest one in
+            # the batch, so each clip's frame count depends on the other
+            # clips it was processed with. Trim each clip to the number of
+            # frames it would have if processed alone so that each item's
+            # output (and thus its cache entry) is independent of the other
+            # items in the batch. `batch_audio_features` re-pads the
+            # per-item features so audio_tower can still run batched.
+            num_frames = [
+                self._get_num_audio_frames_alone(len(audio))
+                for audio in mm_data["audio"]
+            ]
+            processed_data["input_features_padded"] = [
+                features[:n]
+                for features, n in zip(processed_data.pop("input_features"), num_frames)
+            ]
+            processed_data["input_features_mask"] = [
+                mask[:n]
+                for mask, n in zip(processed_data["input_features_mask"], num_frames)
             ]
 
-            # Unpad features here since we need the output of each item to be
-            # independent of other items for the cache to work correctly
-            unpadded_features = [
-                f[mask]
-                for f, mask in zip(
-                    processed_outputs["input_features"],
-                    processed_outputs["input_features_mask"],
-                )
-            ]
-            processed_outputs["input_features"] = unpadded_features
-        return processed_outputs
+        return processed_data
+
+    def _get_num_audio_frames_alone(self, num_samples: int) -> int:
+        """
+        Get the number of mel frames that the feature extractor produces
+        for a single audio clip of the given length, independent of batch
+        padding.
+        """
+        feature_extractor = self.info.get_feature_extractor()
+        call_params = inspect.signature(type(feature_extractor).__call__).parameters
+        max_length = call_params["max_length"].default
+        pad_to_multiple_of = call_params["pad_to_multiple_of"].default
+
+        num_samples = min(num_samples, max_length)
+        remainder = num_samples % pad_to_multiple_of
+        if remainder:
+            num_samples += pad_to_multiple_of - remainder
+
+        frame_size = feature_extractor.frame_length + 1
+        num_frames = (num_samples - frame_size) // feature_extractor.hop_length + 1
+
+        return max(num_frames, 0)
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/gemma4_mm.py

@@ -560,13 +560,22 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
 
 
 class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
+        merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
         val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
 
         if val is not None and val not in _SUPPORTED_SOFT_TOKENS:
@@ -611,15 +620,15 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
                 timestamps = [idx / fps for idx in frame_indices]
 
                 # Process frames as images with max_soft_tokens=70
-                video_mm_kwargs = dict(mm_kwargs)
+                video_mm_kwargs = dict(hf_processor_mm_kwargs)
                 video_mm_kwargs["max_soft_tokens"] = _VIDEO_MAX_SOFT_TOKENS
 
                 dummy_prompt = ("\t" + processor.image_token) * len(frames)
 
-                frame_outputs = super()._call_hf_processor(
-                    prompt=dummy_prompt,
-                    mm_data={"images": frames},
-                    mm_kwargs=video_mm_kwargs,
+                frame_outputs = self.info.ctx.call_hf_processor(
+                    self.info.get_hf_processor(**video_mm_kwargs),
+                    dict(text=dummy_prompt, **{"images": frames}),
+                    video_mm_kwargs,
                 )
 
                 # Remap HF key name
@@ -660,7 +669,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
             # split-replace bug where replacement text (which itself
             # contains <|video|> tokens) collides with later splits.
             vt = processor.video_token
-            parts = prompt.split(vt, len(video_replacements))
+            parts = prompt_text.split(vt, len(video_replacements))
 
             # NOTE: len(parts) <= len(video_replacements) + 1
             parts_with_repl: list[str] = []
@@ -668,7 +677,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
                 parts_with_repl.extend([part, repl])
             parts_with_repl.extend(parts[len(video_replacements) :])
 
-            prompt = "".join(parts_with_repl)
+            prompt_text = "".join(parts_with_repl)
 
             video_outputs = {
                 "pixel_values_videos": torch.cat(all_video_pixel_values, dim=0),
@@ -712,45 +721,46 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
         # NOTE: This requires a corresponding type annotation on the
         # HF side (Gemma4ProcessorKwargs.images_kwargs) so that
         # _merge_kwargs routes max_soft_tokens into images_kwargs.
-        patched_mm_kwargs = dict(mm_kwargs)
+        patched_mm_kwargs = dict(hf_processor_mm_kwargs)
         if val is not None and is_top_level_max_soft_tokens:
             patched_mm_kwargs["max_soft_tokens"] = val
 
-        processed_outputs = super()._call_hf_processor(
-            prompt,
-            mm_data,
+        processed_data = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**patched_mm_kwargs),
+            dict(text=prompt_text, **mm_data),
             patched_mm_kwargs,
         )
 
         # HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'.
         # Remap here to keep a single translation point.
-        if "image_position_ids" in processed_outputs:
-            processed_outputs["pixel_position_ids"] = processed_outputs.pop(
+        if "image_position_ids" in processed_data:
+            processed_data["pixel_position_ids"] = processed_data.pop(
                 "image_position_ids"
             )
 
-        if "input_features" in processed_outputs:
+        if "input_features" in processed_data:
             # Unpad per-item so each item's cache entry is
             # self-contained. The batched() field config in
             # _get_mm_fields_config stacks equal-length items and leaves
             # the rest as a list; _process_audio_input re-pads that list
             # to the batch's max length.
-            masks = processed_outputs["input_features_mask"]
+            masks = processed_data["input_features_mask"]
             unpadded_features = [
                 f[mask]
                 for f, mask in zip(
-                    processed_outputs["input_features"],
+                    processed_data["input_features"],
                     masks,
                 )
             ]
             unpadded_masks = [mask[mask] for mask in masks]
-            processed_outputs["input_features"] = unpadded_features
-            processed_outputs["input_features_padded"] = unpadded_features
-            processed_outputs["input_features_mask"] = unpadded_masks
+            processed_data["input_features"] = unpadded_features
+            processed_data["input_features_padded"] = unpadded_features
+            processed_data["input_features_mask"] = unpadded_masks
 
-        # Merge video outputs into the final result
-        combined_outputs = dict(processed_outputs, **video_outputs)
-        return BatchFeature(combined_outputs)
+        processed_data.update(video_outputs)
+        processed_data.update(passthrough_data)
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,
@@ -815,7 +825,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
                 image_size = images.get_image_size(item_idx)
                 # Resolve the effective max_soft_tokens by merging
                 # per-prompt kwargs with the config-level defaults,
-                # consistent with how _call_hf_processor resolves it.
+                # consistent with how _apply_hf_processor_main resolves it.
                 # Without this merge, a missing per-prompt override
                 # would fall back to vision_cfg.default_output_length
                 # instead of the config's mm_processor_kwargs default.

vllm/model_executor/models/glm4_1v.py

@@ -1576,32 +1576,39 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
 
         return prepared_data, prepared_kwargs
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        mm_data = dict(mm_data)
-        if not mm_data:
-            tokenizer = self.info.get_tokenizer()
-            prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
 
-        processor = self.info.get_hf_processor(**mm_kwargs)
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
+        mm_data = dict(mm_data)
+
+        processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
 
         use_direct_path = (
             not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
         )
         if use_direct_path:
             prepared_data, prepared_kwargs = self._get_direct_path_inputs(
-                mm_data, mm_kwargs
+                mm_data, hf_processor_mm_kwargs
             )
-            return super()._call_hf_processor(
-                prompt=prompt,
-                mm_data=prepared_data,
-                mm_kwargs=prepared_kwargs,
+            processed_data = self.info.ctx.call_hf_processor(
+                self.info.get_hf_processor(**prepared_kwargs),
+                dict(text=prompt_text, **prepared_data),
+                prepared_kwargs,
             )
+            processed_data.update(passthrough_data)
+            return processed_data
 
         if (
             "videos" in mm_data
@@ -1616,7 +1623,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
                 video_array, metadata = item
 
                 # don't update mm_kwargs inplace
-                video_mm_kwargs = dict(**mm_kwargs)
+                video_mm_kwargs = dict(**hf_processor_mm_kwargs)
                 video_mm_kwargs["do_sample_frames"] = metadata.get(
                     "do_sample_frames", True
                 )
@@ -1625,10 +1632,13 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
                 video_mm_data["videos"] = [[video_array]]
                 video_mm_data["video_metadata"] = [[_to_video_metadata(metadata)]]
 
-                video_outputs = super()._call_hf_processor(
-                    prompt="<|begin_of_video|><|video|><|end_of_video|>",
-                    mm_data=video_mm_data,
-                    mm_kwargs=video_mm_kwargs,
+                video_outputs = self.info.ctx.call_hf_processor(
+                    self.info.get_hf_processor(**video_mm_kwargs),
+                    dict(
+                        text="<|begin_of_video|><|video|><|end_of_video|>",
+                        **video_mm_data,
+                    ),
+                    video_mm_kwargs,
                 )
                 input_ids = video_outputs.pop("input_ids")
                 if swap_video_frame_tokens:
@@ -1636,7 +1646,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
                         processor.video_token_id
                     )
                 video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
-                prompt = prompt.replace(
+                prompt_text = prompt_text.replace(
                     "<|begin_of_video|><|video|><|end_of_video|>",
                     video_placeholder,
                     1,
@@ -1652,20 +1662,20 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
             video_outputs = dict()
             swap_video_frame_tokens = False
 
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
+        processed_data = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**hf_processor_mm_kwargs),
+            dict(text=prompt_text, **mm_data),
+            hf_processor_mm_kwargs,
         )
         if swap_video_frame_tokens:
-            input_ids = processed_outputs["input_ids"]
+            input_ids = processed_data["input_ids"]
             input_ids[input_ids == processor.video_token_id] = processor.image_token_id
-            processed_outputs["input_ids"] = input_ids
-        combined_outputs = dict(
-            processed_outputs,
-            **video_outputs,
-        )
-        return BatchFeature(combined_outputs)
+            processed_data["input_ids"] = input_ids
+
+        processed_data.update(video_outputs)
+        processed_data.update(passthrough_data)
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/glmasr.py

@@ -750,46 +750,40 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
             chunk_counts.append(min(n_chunks, max_windows))
         return chunk_counts
 
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
-        mm_data: dict[str, object],
-        mm_kwargs: Mapping[str, Any],
-    ) -> BatchFeature:
-        # Normalize input: handle deprecated key and list conversion.
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        mm_data = dict(mm_data)
         if "audios" in mm_data:
             mm_data["audio"] = mm_data.pop("audios")
 
-        audio = mm_data.get("audio", [])
-        audio_list = [audio] if audio and not isinstance(audio, list) else audio
-
-        # Early return for text-only.
-        if not audio_list:
-            prompt_ids = self.info.get_tokenizer().encode(prompt)
-            prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
-
-        # Handle sampling_rate
-        feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
-        mm_kwargs = dict(
-            **mm_kwargs,
+        feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
+        hf_processor_mm_kwargs = dict(
+            **hf_processor_mm_kwargs,
             sampling_rate=feature_extractor.sampling_rate,
         )
 
-        # Call parent method
-        outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
+        return mm_data, hf_processor_mm_kwargs
 
-        # Postprocess: rename mask and add chunk counts
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
         # Handle different key names from different transformers versions
-        if "input_features_mask" in outputs:
-            outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
-        elif "input_features_mask" not in outputs and "input_features" in outputs:
+        if "input_features_mask" in processed_data:
+            processed_data["feature_attention_mask"] = processed_data.pop(
+                "input_features_mask"
+            )
+        elif "input_features" in processed_data:
             # If no mask is provided, create one from input_features
-            input_features = outputs["input_features"]
+            input_features = processed_data["input_features"]
             if isinstance(input_features, torch.Tensor):
                 # Create a mask of all ones matching the sequence length
                 mask = torch.ones(
@@ -797,18 +791,22 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
                     input_features.shape[-1],
                     dtype=torch.long,
                 )
-                outputs["feature_attention_mask"] = mask
+                processed_data["feature_attention_mask"] = mask
 
-        # Get processor for chunk counts calculation
-        processor = self.info.get_hf_processor(**mm_kwargs)
+        audio = mm_data.get("audio", [])
+        audio_list = [audio] if audio and not isinstance(audio, list) else audio
+        if audio_list:
+            processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
 
-        # Override chunk counts calculation with GLM-ASR specific logic
-        chunk_counts = self._calculate_chunk_counts(
-            audio_list, processor.feature_extractor, processor
-        )
-        outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
+            # Override chunk counts calculation with GLM-ASR specific logic
+            chunk_counts = self._calculate_chunk_counts(
+                audio_list, processor.feature_extractor, processor
+            )
+            processed_data["chunk_counts"] = torch.tensor(
+                chunk_counts, dtype=torch.long
+            )
 
-        return outputs
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/granite_speech.py

@@ -180,12 +180,14 @@ class GraniteSpeechMultiModalProcessor(
             )
         ]
 
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
         mm_data = dict(mm_data)
         audios = mm_data.pop("audios", [])
 
@@ -193,21 +195,23 @@ class GraniteSpeechMultiModalProcessor(
             # GraniteSpeechFeatureExtractor accepts "audio"
             mm_data["audio"] = audios
 
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
+        return mm_data, hf_processor_mm_kwargs
 
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
         if "audio" in mm_data:
             # Calculate the number of audio tokens per entry in the batch;
             # This is used to split the batch back out after padding.
             audio_token_index = self.info.get_hf_config().audio_token_index
-            processed_outputs["audio_embed_sizes"] = (
-                processed_outputs["input_ids"] == audio_token_index
+            processed_data["audio_embed_sizes"] = (
+                processed_data["input_ids"] == audio_token_index
             ).sum(-1)
 
-        return processed_outputs
+        return processed_data
 
 
 class GraniteSpeechDummyInputsBuilder(

vllm/model_executor/models/h2ovl.py

@@ -135,7 +135,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
         self,
         inputs: ProcessorInputs,
         timing_ctx: TimingContext,
-    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
+    ) -> MultiModalProcessingInfo:
         # The processor logic is different for len(images) <= 1 vs > 1
         # Since the processing cache assumes that the processor output is
         # invariant of how many images are passed per prompt, we only

vllm/model_executor/models/hunyuan_vision.py

@@ -731,28 +731,6 @@ class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo
 
 
 class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        hf_processor = self.info.get_hf_processor(**mm_kwargs)
-        # HunYuanVLProcessor requires image placeholders wrapped with start/end tokens.
-        if mm_data.get("images") is not None and prompt:
-            img_tok = hf_processor.image_token
-            wrapped = (
-                f"{hf_processor.image_start_token}{img_tok}"
-                f"{hf_processor.image_end_token}"
-            )
-            if img_tok in prompt and wrapped not in prompt:
-                prompt = prompt.replace(img_tok, wrapped)
-        return self.info.ctx.call_hf_processor(
-            hf_processor,
-            dict(text=prompt, **mm_data),
-            mm_kwargs,
-        )
-
     def _get_prompt_updates(
         self,
         mm_items: MultiModalDataItems,

vllm/model_executor/models/hyperclovax_vision.py

@@ -190,77 +190,71 @@ class HCXVisionDummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionProcessingInfo
 
 
 class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingInfo]):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
         for video_idx, video_arr in enumerate(mm_data.get("videos", [])):
             if video_arr.dtype != np.uint8:
                 mm_data["videos"][video_idx] = video_arr.astype(np.uint8)
 
-        processed_outputs = self.info.ctx.call_hf_processor(
-            hf_processor=self.info.get_hf_processor(**mm_kwargs),
+        images = mm_data.get("images")
+        videos = mm_data.get("videos")
+
+        # batchify input as a single item
+        processed_data = self.info.ctx.call_hf_processor(
+            hf_processor=self.info.get_hf_processor(**hf_processor_mm_kwargs),
             data=dict(
-                text=prompt,
-                images=None,
-                videos=None,
+                images=None if images is None else [images],
+                videos=None if videos is None else [videos],
             ),
-        )  # text-only
+        )
 
-        if len(mm_data) > 0:
-            images = mm_data.get("images")
-            videos = mm_data.get("videos")
+        for k, v in processed_data.items():
+            if isinstance(v, list) and len(v) > 0:
+                assert len(v) == 1
+                processed_data[k] = v[0]
 
-            # batchify input as a single item
-            _processed_outputs = self.info.ctx.call_hf_processor(
-                hf_processor=self.info.get_hf_processor(**mm_kwargs),
-                data=dict(
-                    text=None,
-                    images=None if images is None else [images],
-                    videos=None if videos is None else [videos],
-                ),
-            )  # mm-only
+        if images:
+            processed_data["image_sizes_images"] = torch.tensor(
+                processed_data["image_sizes_images"]
+            )
+            processed_data["vision_query_lengths_images"] = torch.tensor(
+                processed_data["vision_query_lengths_images"]
+            )
 
-            for k, v in _processed_outputs.items():
-                if isinstance(v, list) and len(v) > 0:
-                    assert len(v) == 1
-                    _processed_outputs[k] = v[0]
-
-            if images:
-                _processed_outputs["image_sizes_images"] = torch.tensor(
-                    _processed_outputs["image_sizes_images"]
-                )
-                _processed_outputs["vision_query_lengths_images"] = torch.tensor(
-                    _processed_outputs["vision_query_lengths_images"]
-                )
-
-            if videos:
-                _idx_per_video = [
-                    0,
-                    *accumulate(
-                        get_num_combined_frames(len(video)) for video in videos
-                    ),
+        if videos:
+            _idx_per_video = [
+                0,
+                *accumulate(get_num_combined_frames(len(video)) for video in videos),
+            ]
+            processed_data["pixel_values_videos"] = [
+                processed_data["pixel_values_videos"][
+                    _idx_per_video[i] : _idx_per_video[i + 1]
                 ]
-                _processed_outputs["pixel_values_videos"] = [
-                    _processed_outputs["pixel_values_videos"][
+                for i in range(len(videos))
+            ]
+            processed_data["vision_query_lengths_videos"] = [
+                torch.tensor(
+                    processed_data["vision_query_lengths_videos"][
                         _idx_per_video[i] : _idx_per_video[i + 1]
                     ]
-                    for i in range(len(videos))
-                ]
-                _processed_outputs["vision_query_lengths_videos"] = [
-                    torch.tensor(
-                        _processed_outputs["vision_query_lengths_videos"][
-                            _idx_per_video[i] : _idx_per_video[i + 1]
-                        ]
-                    )
-                    for i in range(len(videos))
-                ]
+                )
+                for i in range(len(videos))
+            ]
 
-            processed_outputs.update(_processed_outputs)
+        processed_data.update(passthrough_data)
 
-        return processed_outputs
+        return processed_data
 
     def _get_prompt_updates(
         self,

vllm/model_executor/models/hyperclovax_vision_v2.py

@@ -261,34 +261,44 @@ class HCXVisionV2MultiModalProcessor(
 ):
     """Multimodal processor for HyperCLOVAX V2 (32B Think model)."""
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
         images = mm_data.get("images")
         videos = mm_data.get("videos")
 
         # Get the HF processor
-        hf_processor = self.info.get_hf_processor(**mm_kwargs)
+        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
 
         # Build data dict for HF processor (images/videos only)
         # NOTE: We pass the prompt as-is without token normalization.
         # Token expansion is handled by vLLM via _get_prompt_updates.
         data: dict[str, object] = dict(
-            text=prompt,
+            text=prompt_text,
             images=images,
             videos=videos,
         )
 
-        processed_outputs = self.info.ctx.call_hf_processor(
+        processed_data = self.info.ctx.call_hf_processor(
             hf_processor=hf_processor,
             data=data,
-            kwargs=mm_kwargs,
+            kwargs=hf_processor_mm_kwargs,
         )
+        processed_data.update(passthrough_data)
 
-        return processed_outputs
+        return processed_data
 
     def _get_prompt_updates(
         self,

vllm/model_executor/models/idefics3.py

@@ -239,51 +239,58 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
 
 
 class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        # Text-only input not supported in composite processor
-        if not (images := mm_data.get("images", [])):
-            prompt_ids = self.info.get_tokenizer().encode(prompt)
-            prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
 
         image_processor = self.info.get_hf_processor().image_processor
         if getattr(image_processor, "backend", "pil") == "pil":
-            mm_kwargs = {"input_data_format": "channels_last", **mm_kwargs}
+            hf_processor_mm_kwargs = {
+                "input_data_format": "channels_last",
+                **hf_processor_mm_kwargs,
+            }
 
-        processed_outputs = super()._call_hf_processor(
-            prompt,
+        processed_data = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**hf_processor_mm_kwargs),
             mm_data,
-            mm_kwargs,
+            hf_processor_mm_kwargs,
         )
 
+        images = mm_data.get("images", [])
         mm_items = self.info.parse_mm_data({"image": images}, validate=False)
         parsed_images = mm_items.get_items("image", ImageProcessorItems)
         image_sizes = [
             parsed_images.get_image_size(i) for i in range(len(parsed_images))
         ]
-        hf_processor = self.info.get_hf_processor(**mm_kwargs)
+        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
 
         num_patches = [
             self.info.get_num_patches(
                 image_width=size.width,
                 image_height=size.height,
                 processor=hf_processor,
-                mm_kwargs=mm_kwargs,
+                mm_kwargs=hf_processor_mm_kwargs,
             )
             for size in image_sizes
         ]
-        processed_outputs["num_patches"] = torch.tensor(num_patches)
+        processed_data["num_patches"] = torch.tensor(num_patches)
 
         # Remove the extra batch dimension
-        processed_outputs["pixel_values"].squeeze_(0)
-        processed_outputs["pixel_attention_mask"].squeeze_(0)
+        processed_data["pixel_values"].squeeze_(0)
+        processed_data["pixel_attention_mask"].squeeze_(0)
 
-        return processed_outputs
+        processed_data.update(passthrough_data)
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/interns1.py

@@ -332,19 +332,28 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
 class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]):
     """Basic image-only MultiModalProcessor for InternS1-style models."""
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
         mm_data = dict(mm_data)
         videos = mm_data.pop("videos", [])
         images = mm_data.pop("images", [])
         assert isinstance(videos, list)
         assert isinstance(images, list)
 
-        hf_processor = self.info.get_hf_processor(**mm_kwargs)
+        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
         tokenizer = hf_processor.tokenizer
         video_token_id = tokenizer.encode(
             hf_processor.video_token, add_special_tokens=False
@@ -352,23 +361,29 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
         assert len(video_token_id) == 1
         video_token_id = video_token_id[0]
 
-        prompt = re.sub(hf_processor.image_token, "<image_placeholder>", prompt)
-        prompt = re.sub(hf_processor.video_token, "<video_placeholder>", prompt)
+        prompt_text = re.sub(
+            hf_processor.image_token, "<image_placeholder>", prompt_text
+        )
+        prompt_text = re.sub(
+            hf_processor.video_token, "<video_placeholder>", prompt_text
+        )
 
         image_outputs = {}
         if images:
             image_pixel_values = []
             for image in images:
-                processed_outputs = super()._call_hf_processor(
-                    prompt=hf_processor.image_token,
-                    mm_data={"images": image},
-                    mm_kwargs=mm_kwargs,
+                processed_data = self.info.ctx.call_hf_processor(
+                    self.info.get_hf_processor(**hf_processor_mm_kwargs),
+                    dict(text=hf_processor.image_token, **{"images": image}),
+                    hf_processor_mm_kwargs,
                 )
-                image_pixel_values.append(processed_outputs.pop("pixel_values"))
+                image_pixel_values.append(processed_data.pop("pixel_values"))
 
-                input_ids = processed_outputs.pop("input_ids")
+                input_ids = processed_data.pop("input_ids")
                 image_placeholder = tokenizer.batch_decode(input_ids)[0]
-                prompt = prompt.replace("<image_placeholder>", image_placeholder, 1)
+                prompt_text = prompt_text.replace(
+                    "<image_placeholder>", image_placeholder, 1
+                )
 
             num_patches = [len(item) for item in image_pixel_values]
             image_outputs = {
@@ -381,18 +396,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
         if videos:
             video_pixel_values = []
             for video in videos:
-                processed_outputs = super()._call_hf_processor(
-                    prompt=hf_processor.video_token,
-                    mm_data={"videos": video},
-                    mm_kwargs=mm_kwargs,
+                processed_data = self.info.ctx.call_hf_processor(
+                    self.info.get_hf_processor(**hf_processor_mm_kwargs),
+                    dict(text=hf_processor.video_token, **{"videos": video}),
+                    hf_processor_mm_kwargs,
                 )
-                video_pixel_values.append(processed_outputs.pop("pixel_values"))
+                video_pixel_values.append(processed_data.pop("pixel_values"))
 
-                input_ids = processed_outputs.pop("input_ids")
+                input_ids = processed_data.pop("input_ids")
                 input_ids[input_ids == hf_processor.image_token_id] = video_token_id
 
                 video_placeholder = tokenizer.batch_decode(input_ids)[0]
-                prompt = prompt.replace("<video_placeholder>", video_placeholder, 1)
+                prompt_text = prompt_text.replace(
+                    "<video_placeholder>", video_placeholder, 1
+                )
 
             num_frames = [len(item) for item in video_pixel_values]
             video_outputs = {
@@ -401,11 +418,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
                 "video_token_id": torch.tensor(video_token_id),
             }
 
-        prompt = re.sub("<image_placeholder>", hf_processor.image_token, prompt)
-        prompt = re.sub("<video_placeholder>", hf_processor.video_token, prompt)
-        text_outputs = tokenizer(prompt, return_tensors="pt")
+        prompt_text = re.sub(
+            "<image_placeholder>", hf_processor.image_token, prompt_text
+        )
+        prompt_text = re.sub(
+            "<video_placeholder>", hf_processor.video_token, prompt_text
+        )
+        text_outputs = tokenizer(prompt_text, return_tensors="pt")
 
-        return BatchFeature({**text_outputs, **image_outputs, **video_outputs})
+        processed_data.update(text_outputs)
+        processed_data.update(image_outputs)
+        processed_data.update(video_outputs)
+        processed_data.update(passthrough_data)
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/internvl.py

@@ -220,27 +220,21 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
 class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
     """Basic image-only MultiModalProcessor for InternVL-style models."""
 
-    def _call_hf_processor(
+    def _postprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
     ) -> BatchFeature:
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
-
-        hf_processor = self.info.get_hf_processor(**mm_kwargs)
+        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
         image_token_id = hf_processor.ctx_image_token_id
 
         # Since there may be extra tokens in the feature placeholders,
         # we need to pass the image token ID to the model to select the
         # tokens to merge from the vision encoder outputs
-        processed_outputs["image_token_id"] = torch.tensor(image_token_id)
+        processed_data["image_token_id"] = torch.tensor(image_token_id)
 
-        return processed_outputs
+        return processed_data
 
     def _get_image_fields_config(self, hf_inputs: BatchFeature):
         image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
@@ -455,19 +449,23 @@ class InternVLMultiModalProcessor(
 ):
     """InternVL MultiModalProcessor extended for video support"""
 
-    def _call_hf_processor(
+    def _postprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
     ) -> BatchFeature:
-        processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
+        processed_data = super()._postprocess_hf_mm_data(
+            mm_data,
+            hf_processor_mm_kwargs,
+            processed_data,
+        )
 
-        hf_processor = self.info.get_hf_processor(**mm_kwargs)
+        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
         if (video_token_id := hf_processor.ctx_video_token_id) is not None:
-            processed_outputs["video_token_id"] = torch.tensor(video_token_id)
+            processed_data["video_token_id"] = torch.tensor(video_token_id)
 
-        return processed_outputs
+        return processed_data
 
     def _get_video_fields_config(self, hf_inputs: BatchFeature):
         video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))

vllm/model_executor/models/jina_vl.py

@@ -12,6 +12,7 @@ from vllm.logger import init_logger
 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
 from vllm.model_executor.layers.pooler import DispatchPooler
 from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.parse import MultiModalDataItems
 from vllm.sequence import IntermediateTensors
 
 from .interfaces import SupportsCrossEncoding, SupportsMultiModal, SupportsScoreTemplate
@@ -54,19 +55,28 @@ class JinaVLScorer(nn.Module):
 
 
 class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        # NOTE: We should reverse the order of the mm_data because the
-        # query prompt is placed after the document prompt in the score
-        # template for JinaVLForRanking model, but in mm_data they are
-        # stored in the opposite order (query first, then document).
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
         for _, value in mm_data.items():
             value.reverse()
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
+        processed_data = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**hf_processor_mm_kwargs),
+            mm_data,
+            hf_processor_mm_kwargs,
+        )
+        processed_data.update(passthrough_data)
+        return processed_data
 
 
 @MULTIMODAL_REGISTRY.register_processor(

vllm/model_executor/models/kanana_v.py

@@ -461,17 +461,21 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
     def media_token_id(self) -> int:
         return self.info.get_hf_config().text_config.eos_token_id + 1
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
         """Run the underlying HF processor on text and image data."""
-        # Text-only input is handled as a special case here.
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
         if not mm_data or not mm_data.get("images", []):
-            prompt_ids = self.info.get_tokenizer().encode(prompt)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
 
         # Images
         image_inputs = mm_data.get("images", [])
@@ -493,7 +497,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
 
         tokenizer = self.info.get_tokenizer()
         media_token = tokenizer.convert_ids_to_tokens([self.media_token_id])[0]
-        prompt_replaced = prompt.replace("<image>", media_token)
+        prompt_replaced = prompt_text.replace("<image>", media_token)
         input_ids = tokenizer.encode(prompt_replaced)
         input_ids = torch.tensor(input_ids)
 
@@ -527,7 +531,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
             image_token_thw=torch.tensor(image_meta["image_token_thw"]),
             pixel_sizes=torch.tensor(pixel_sizes),
         )
-        return BatchFeature(combined_outputs, tensor_type="pt")
+        processed_data = BatchFeature(combined_outputs, tensor_type="pt")
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_prompt_updates(
         self,

vllm/model_executor/models/keye.py

@@ -1162,15 +1162,8 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):
 
 
 class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        # Override to use the text path instead of token path to use the
-        # video-specific logic in processing_keye.py
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
 
     def _get_prompt_updates(
         self,

vllm/model_executor/models/keye_vl1_5.py

@@ -378,15 +378,8 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
 
 
 class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        # Override to use the text path instead of token path to use the
-        # video-specific logic in processing_keye.py
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
 
     def _get_prompt_updates(
         self,

vllm/model_executor/models/kimi_audio.py

@@ -37,6 +37,7 @@ from vllm.multimodal.parse import (
     DictEmbeddingItems,
     ModalityData,
     ModalityDataItems,
+    MultiModalDataItems,
     MultiModalDataParser,
 )
 from vllm.multimodal.processing import (
@@ -226,13 +227,20 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser):
 class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]):
     """vLLM multi-modal processor wrapper for Kimi-Audio."""
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
         """Call the HuggingFace processor."""
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
         # Convert mm_data format: {'audios': [...]} -> {'audio': ...}
         mm_data = dict(mm_data)
         audios = mm_data.pop("audios", [])
@@ -252,11 +260,13 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
             mm_data["audio"] = audio_arrays
 
         # Use the context's call_hf_processor for proper handling
-        return self.info.ctx.call_hf_processor(
-            self.info.get_hf_processor(**mm_kwargs),
-            dict(text=prompt, **mm_data),
-            mm_kwargs,
+        processed_data = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**hf_processor_mm_kwargs),
+            mm_data,
+            hf_processor_mm_kwargs,
         )
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/kimi_k25.py

@@ -258,16 +258,6 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
             grid_thws=MultiModalFieldConfig.batched("vision_chunk", keep_on_cpu=True),
         )
 
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        # Override to use the text path instead of token path because vision chunk
-        # is not considered
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
-
     def _get_prompt_updates(
         self,
         mm_items: MultiModalDataItems,

vllm/model_executor/models/kimi_vl.py

@@ -237,6 +237,9 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
 
 
 class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
     def _get_mm_fields_config(
         self,
         hf_inputs: BatchFeature,

vllm/model_executor/models/lfm2_vl.py

@@ -393,45 +393,38 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):
 
 
 class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _postprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
     ) -> BatchFeature:
-        # Text-only input not supported in composite processor
-        if not (images := mm_data.get("images", [])):
-            prompt_ids = self.info.get_tokenizer().encode(
-                prompt, add_special_tokens=False
-            )
-            prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
-
-        processed_outputs = super()._call_hf_processor(
-            prompt,
-            mm_data,
-            mm_kwargs,
-        )
+        if not mm_data:
+            return processed_data
 
+        images = mm_data.get("images", [])
         mm_items = self.info.parse_mm_data({"image": images}, validate=False)
         parsed_images = mm_items.get_items("image", ImageProcessorItems)
         image_sizes = [
             parsed_images.get_image_size(i) for i in range(len(parsed_images))
         ]
-        hf_processor = self.info.get_hf_processor(**mm_kwargs)
+        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
 
         num_patches = [
             self.info.get_num_patches(
                 image_width=size.width,
                 image_height=size.height,
                 processor=hf_processor,
-                mm_kwargs=mm_kwargs,
+                mm_kwargs=hf_processor_mm_kwargs,
             )
             for size in image_sizes
         ]
-        processed_outputs["num_patches"] = torch.tensor(num_patches)
+        processed_data["num_patches"] = torch.tensor(num_patches)
 
-        return processed_outputs
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/lightonocr.py

@@ -39,20 +39,19 @@ _I = TypeVar("_I", bound=Mistral3ProcessingInfo)
 
 
 class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
 
-        # NOTE: LightOnOCR does not use break/end tokens, so we remove them here.
-        input_ids = processed_outputs.get("input_ids")
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if not mm_data:
+            return processed_data
+
+        input_ids = processed_data.get("input_ids")
         if input_ids is not None:
             processor = self.info.get_hf_processor()
             tokenizer = self.info.get_tokenizer()
@@ -67,22 +66,22 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
                 torch.tensor([break_id, end_id]),
             )
 
-            processed_outputs["input_ids"] = input_ids[keep_mask].unsqueeze(0)
-            if "attention_mask" in processed_outputs:
-                processed_outputs["attention_mask"] = processed_outputs[
-                    "attention_mask"
-                ][keep_mask].unsqueeze(0)
+            processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0)
+            if "attention_mask" in processed_data:
+                processed_data["attention_mask"] = processed_data["attention_mask"][
+                    keep_mask
+                ].unsqueeze(0)
 
         # un-pad pixel_values per-image so caches remain independent.
-        pixel_values = processed_outputs.get("pixel_values")
+        pixel_values = processed_data.get("pixel_values")
         if pixel_values is not None:
-            image_sizes = processed_outputs["image_sizes"]
+            image_sizes = processed_data["image_sizes"]
             assert len(pixel_values) == len(image_sizes)
-            processed_outputs["pixel_values"] = [
+            processed_data["pixel_values"] = [
                 p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
             ]
 
-        return processed_outputs
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/llava.py

@@ -321,30 +321,30 @@ class PixtralHFProcessingInfo(BaseLlavaProcessingInfo):
 
 
 class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
 
-        pixel_values = processed_outputs.get("pixel_values")
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if not mm_data:
+            return processed_data
+
+        pixel_values = processed_data.get("pixel_values")
         if pixel_values is not None:
             # Avoid padding since we need the output for each image to be
             # independent of other images for the cache to work correctly
-            image_sizes = processed_outputs["image_sizes"]
+            image_sizes = processed_data["image_sizes"]
             assert len(pixel_values) == len(image_sizes)
 
-            processed_outputs["pixel_values"] = [
+            processed_data["pixel_values"] = [
                 p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
             ]
 
-        return processed_outputs
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/llava_next.py

@@ -211,6 +211,9 @@ class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]):
 
 
 class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
     def _get_mm_fields_config(
         self,
         hf_inputs: BatchFeature,

vllm/model_executor/models/llava_onevision.py

@@ -321,22 +321,33 @@ class LlavaOnevisionMultiModalProcessor(
             pixel_values_videos=MultiModalFieldConfig.batched("video"),
         )
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
         mm_data = dict(mm_data)
         videos = mm_data.pop("videos", [])
         assert isinstance(videos, list)
 
         if not videos:
-            return super()._call_hf_processor(
-                prompt=prompt,
-                mm_data=mm_data,
-                mm_kwargs=mm_kwargs,
+            processed_data = self.info.ctx.call_hf_processor(
+                self.info.get_hf_processor(**hf_processor_mm_kwargs),
+                dict(text=prompt_text, **mm_data),
+                hf_processor_mm_kwargs,
             )
+            processed_data.update(passthrough_data)
+            return processed_data
 
         # LLaVA-OneVision processor doesn't support multiple videos
         # with different sizes when converting back to tensors
@@ -346,19 +357,13 @@ class LlavaOnevisionMultiModalProcessor(
         image_token = processor.image_token
         video_token = processor.video_token
 
-        text_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data={},
-            mm_kwargs=mm_kwargs,
-        )
-
         images = mm_data.pop("images", [])
         assert isinstance(images, list)
         if images:
-            processor_outputs = super()._call_hf_processor(
-                prompt=image_token * len(images),
-                mm_data={"images": images},
-                mm_kwargs=mm_kwargs,
+            processor_outputs = self.info.ctx.call_hf_processor(
+                self.info.get_hf_processor(**hf_processor_mm_kwargs),
+                dict(text=image_token * len(images), **{"images": images}),
+                hf_processor_mm_kwargs,
             )
             image_outputs = {
                 k: v
@@ -370,10 +375,10 @@ class LlavaOnevisionMultiModalProcessor(
 
         pixel_values_videos = []
         for video in videos:
-            item_outputs = super()._call_hf_processor(
-                prompt=video_token,
-                mm_data={"videos": video},
-                mm_kwargs=mm_kwargs,
+            item_outputs = self.info.ctx.call_hf_processor(
+                self.info.get_hf_processor(**hf_processor_mm_kwargs),
+                dict(text=video_token, **{"videos": video}),
+                hf_processor_mm_kwargs,
             )
 
             pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
@@ -381,11 +386,13 @@ class LlavaOnevisionMultiModalProcessor(
         video_outputs = {"pixel_values_videos": pixel_values_videos}
 
         combined_outputs = dict(
-            text_outputs,
+            input_ids=prompt_text,
             **image_outputs,
             **video_outputs,
         )
-        return BatchFeature(combined_outputs)
+        processed_data = BatchFeature(combined_outputs)
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_prompt_updates(
         self,

vllm/model_executor/models/llava_onevision2.py

@@ -286,7 +286,7 @@ def _validate_video_sources(paths, model_config) -> list[str]:
 #   participates in the standard decode-stage pipeline.
 #
 # * codec backend: NOT a loader. OV2's codec path needs the video path string
-#   to survive into ``_call_hf_processor``, where the HF processor builds the
+#   to survive into ``_apply_hf_processor_main``, where the HF processor builds the
 #   codec canvas + smart_resize + patchify
 #   (pixel_values/image_grid_thw/patch_positions). That transform is
 #   path-level and inseparable; it cannot be reconstructed from pre-decoded
@@ -302,7 +302,7 @@ def prepare_codec_video_input(video_path: str) -> tuple:
 
     Returns ``(dummy_ndarray, metadata)`` where the ndarray satisfies the
     parser's 4-D shape check and the metadata carries the actual path to
-    our ``_call_hf_processor``. Use as::
+    our ``_apply_hf_processor_main``. Use as::
 
         multi_modal_data = {"video": prepare_codec_video_input("foo.mp4")}
 
@@ -1283,7 +1283,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
         # ``video_needs_metadata=True`` makes the parser preserve both the
         # ``(frames, metadata)`` tuples from the frame backend and the
         # ``(dummy, {marker: path})`` tuples from prepare_codec_video_input;
-        # both are dispatched by metadata content in ``_call_hf_processor``.
+        # both are dispatched by metadata content in ``_apply_hf_processor_main``.
         return LlavaOnevision2MultiModalDataParser(
             self.get_hf_config().vision_config.spatial_merge_size,
             video_needs_metadata=True,
@@ -1547,18 +1547,23 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
 class LlavaOnevision2MultiModalProcessor(
     BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]
 ):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        # The wrapped OV2 processor is a bare custom class without the standard
-        # ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path
-        # fails; overriding this method routes the base class to call us
-        # directly.
-        hf_processor = self.info.get_hf_processor(**mm_kwargs)
-        merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
+        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
+        merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
         merged_kwargs.setdefault("return_tensors", "pt")
         call_kwargs = {
             k: v
@@ -1627,21 +1632,25 @@ class LlavaOnevision2MultiModalProcessor(
                 if len(codec_video_paths) > 1
                 else codec_video_paths[0]
             )
-            # Route through the base ``_call_hf_processor`` so float-tensor
+            # Route through ``ctx.call_hf_processor`` so float-tensor
             # dtype postprocessing is applied automatically; inject
             # ``video_backend="codec"`` via mm_kwargs so the wrapped processor
             # dispatches to its codec branch.
-            output = super()._call_hf_processor(
-                prompt=prompt,
-                mm_data=mm_data,
-                mm_kwargs={**mm_kwargs, "video_backend": "codec"},
+            output = self.info.ctx.call_hf_processor(
+                self.info.get_hf_processor(
+                    **{**hf_processor_mm_kwargs, "video_backend": "codec"}
+                ),
+                dict(text=prompt_text, **mm_data),
+                {**hf_processor_mm_kwargs, "video_backend": "codec"},
             )
             data = dict(output)
-            return BatchFeature(
+            processed_data = BatchFeature(
                 self._rename_codec_outputs_to_video(
                     data, codec_video_paths, hf_processor
                 )
             )
+            processed_data.update(passthrough_data)
+            return processed_data
 
         # ---- Frame backend (registered LlavaOnevision2VideoBackend) ------
         # Every non-codec video reaches here as a ``(frames_ndarray, metadata)``
@@ -1657,7 +1666,9 @@ class LlavaOnevision2MultiModalProcessor(
         # local-file gating is enforced by the connector before decoding.
         if videos_present:
             timestamp_decimals = int(
-                mm_kwargs.get("timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS)
+                hf_processor_mm_kwargs.get(
+                    "timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
+                )
             )
 
             per_video_frames: list[list[Image.Image]] = []
@@ -1671,7 +1682,7 @@ class LlavaOnevision2MultiModalProcessor(
             # ``<{t} seconds><|vision_start|><|image_pad|><|vision_end|>``
             # blocks (matches the OV2 hf-chat reference exactly).
             new_prompt = _expand_video_markers_in_prompt(
-                prompt,
+                prompt_text,
                 per_video_timestamps,
                 timestamp_decimals=timestamp_decimals,
             )
@@ -1700,7 +1711,7 @@ class LlavaOnevision2MultiModalProcessor(
             row_is_video: list[bool] = []
             vid_idx = 0
             img_idx = 0
-            for marker in marker_pattern.finditer(prompt):
+            for marker in marker_pattern.finditer(prompt_text):
                 if marker.lastgroup == "video":
                     frames = per_video_frames[vid_idx]
                     vid_idx += 1
@@ -1723,15 +1734,15 @@ class LlavaOnevision2MultiModalProcessor(
             merged_mm_data.pop("videos", None)
             merged_mm_data["images"] = flat_frames
 
-            # Route through the base ``_call_hf_processor`` (applies float-tensor
+            # Route through ``ctx.call_hf_processor`` (applies float-tensor
             # dtype postprocessing automatically). The wrapped processor's image
             # branch ignores video/codec-only kwargs and does not forward extra
             # **kwargs to the image processor, so passing the full merged kwarg
             # set here is a no-op beyond return_tensors/padding.
-            output = super()._call_hf_processor(
-                prompt=new_prompt,
-                mm_data=merged_mm_data,
-                mm_kwargs=mm_kwargs,
+            output = self.info.ctx.call_hf_processor(
+                self.info.get_hf_processor(**hf_processor_mm_kwargs),
+                dict(text=new_prompt, **merged_mm_data),
+                hf_processor_mm_kwargs,
             )
             data = dict(output)
 
@@ -1788,17 +1799,21 @@ class LlavaOnevision2MultiModalProcessor(
                 (len(per_video_timestamps),), dtype=torch.long
             )
 
-            return BatchFeature(data)
+            processed_data = BatchFeature(data)
+            processed_data.update(passthrough_data)
+            return processed_data
 
         # ---- Image-only / text-only call --------------------------------
-        # No videos present: delegate to the base ``_call_hf_processor``, which
+        # No videos present: delegate to ``ctx.call_hf_processor``, which
         # runs the wrapped processor over the (possibly empty) image set and
         # applies float-tensor dtype postprocessing automatically.
-        return super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
+        processed_data = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**hf_processor_mm_kwargs),
+            dict(text=prompt_text, **mm_data),
+            hf_processor_mm_kwargs,
         )
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _rename_codec_outputs_to_video(
         self,

vllm/model_executor/models/midashenglm.py

@@ -26,7 +26,7 @@
 import collections
 import collections.abc
 from collections.abc import Callable, Iterable, Mapping, Sequence
-from typing import Annotated, Any, TypeAlias, cast
+from typing import Annotated, TypeAlias, cast
 
 import numpy as np
 import torch
@@ -583,15 +583,17 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
 class MiDashengLMMultiModalProcessor(
     BaseMultiModalProcessor[MiDashengLMProcessingInfo]
 ):
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, Any],
-    ) -> BatchFeature:
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        mm_data = dict(mm_data)
         audios = mm_data.pop("audios", [])
 
-        # + Padding
         min_audio_len = self.info.get_min_audio_len()
         processed_audios = [
             np.pad(
@@ -608,20 +610,7 @@ class MiDashengLMMultiModalProcessor(
         if processed_audios:
             mm_data["audio"] = processed_audios
 
-        if not mm_data.get("audio", []):
-            prompt_ids = self.info.get_tokenizer().encode(prompt)
-            prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
-
-        mm_kwargs = dict(
-            **mm_kwargs,
-        )
-
-        return super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
+        return mm_data, hf_processor_mm_kwargs
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/mimo_v2_omni.py

@@ -912,16 +912,23 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
             fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")
         return fields
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
         """Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.
         Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses
         the singular form.
         """
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
         # Remap audios → audio (MiMoOmniProcessor uses singular param name)
         if "audios" in mm_data:
             mm_data = {**mm_data, "audio": mm_data["audios"]}
@@ -999,7 +1006,13 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
 
             mm_data = {**mm_data, "videos": converted}
 
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
+        processed_data = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**hf_processor_mm_kwargs),
+            mm_data,
+            hf_processor_mm_kwargs,
+        )
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_prompt_updates(
         self,

vllm/model_executor/models/minicpmo.py

@@ -448,7 +448,7 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
         if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems):
             audio_inputs = {}
         else:
-            audio_inputs = self._base_call_hf_processor(
+            audio_inputs = self._call_hf_processor_on_prompts(
                 prompts=[self.info.audio_pattern] * len(parsed_audios),
                 mm_data={"audios": [[audio] for audio in parsed_audios]},
                 mm_kwargs={**mm_kwargs, "chunk_input": True},

vllm/model_executor/models/minicpmv.py

@@ -881,7 +881,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
         if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
             image_inputs = {}
         else:
-            image_inputs = self._base_call_hf_processor(
+            image_inputs = self._call_hf_processor_on_prompts(
                 prompts=[self.info.image_pattern] * len(parsed_images),
                 mm_data={"images": [[image] for image in parsed_images]},
                 mm_kwargs=mm_kwargs,
@@ -906,7 +906,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
         if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
             video_inputs = {}
         else:
-            video_inputs = self._base_call_hf_processor(
+            video_inputs = self._call_hf_processor_on_prompts(
                 prompts=[
                     self.info.image_pattern * len(video) for video in parsed_videos
                 ],
@@ -983,7 +983,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
 
         return new_token_ids, placeholders
 
-    def _base_call_hf_processor(
+    def _call_hf_processor_on_prompts(
         self,
         prompts: list[str],
         mm_data: Mapping[str, Sequence[object]],
@@ -993,19 +993,19 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
     ) -> dict[str, NestedTensors]:
         # This processor supports zipping prompt and mm_data together
         if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}:
-            inputs = super()._call_hf_processor(
-                prompt=prompts,  # type: ignore
-                mm_data=mm_data,
-                mm_kwargs=mm_kwargs,
+            inputs = self.info.ctx.call_hf_processor(
+                self.info.get_hf_processor(**mm_kwargs),
+                dict(text=prompts, **mm_data),
+                mm_kwargs,
             )
         else:
             inputs = defaultdict[str, list[torch.Tensor]](list)
 
             for i, prompt in enumerate(prompts):
-                inputs_one = super()._call_hf_processor(
-                    prompt=prompt,
-                    mm_data={k: v[i] for k, v in mm_data.items()},
-                    mm_kwargs=mm_kwargs,
+                inputs_one = self.info.ctx.call_hf_processor(
+                    self.info.get_hf_processor(**mm_kwargs),
+                    dict(text=prompt, **{k: v[i] for k, v in mm_data.items()}),
+                    mm_kwargs,
                 )
 
                 for k, v in inputs_one.items():
@@ -1014,23 +1014,31 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
 
         return {k: inputs[k] for k in out_keys}
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
         tokenizer = self.info.get_tokenizer()
 
-        input_ids = torch.tensor([tokenizer.encode(prompt)])
-        mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs)
+        input_ids = torch.tensor([tokenizer.encode(prompt_text)])
+        mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
 
-        return BatchFeature(
+        processed_data = BatchFeature(
             {
                 "input_ids": input_ids,
                 **mm_inputs,
             }
         )
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_prompt_updates(
         self,

vllm/model_executor/models/mistral3.py

@@ -233,30 +233,30 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo])
 
 
 class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
 
-        pixel_values = processed_outputs.get("pixel_values")
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if not mm_data:
+            return processed_data
+
+        pixel_values = processed_data.get("pixel_values")
         if pixel_values is not None:
             # Avoid padding since we need the output for each image to be
             # independent of other images for the cache to work correctly
-            image_sizes = processed_outputs["image_sizes"]
+            image_sizes = processed_data["image_sizes"]
             assert len(pixel_values) == len(image_sizes)
 
-            processed_outputs["pixel_values"] = [
+            processed_data["pixel_values"] = [
                 p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
             ]
 
-        return processed_outputs
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/mllama4.py

@@ -589,23 +589,23 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
 
 
 class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
 
-        processor = self.info.get_hf_processor(**mm_kwargs)
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if not mm_data:
+            return processed_data
+
+        processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
         image_processor = processor.image_processor
         vision_config = self.info.get_hf_config().vision_config
 
-        if processed_outputs.get("pixel_values") is not None:
+        if processed_data.get("pixel_values") is not None:
             assert "images" in mm_data, (
                 "images expected to be in mm_data when pixel_values is present"
             )
@@ -636,10 +636,10 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
                 1 if r_h * r_w == 1 else 1 + r_h * r_w for (r_h, r_w) in aspect_ratios
             ]
 
-            processed_outputs["aspect_ratios"] = torch.tensor(aspect_ratios)
-            processed_outputs["patches_per_image"] = torch.tensor(patches_per_image)
+            processed_data["aspect_ratios"] = torch.tensor(aspect_ratios)
+            processed_data["patches_per_image"] = torch.tensor(patches_per_image)
 
-        return processed_outputs
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/molmo.py

@@ -1123,17 +1123,44 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
 
 
 class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
-    def _call_hf_processor(
+    def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
+        processor = self.info.get_hf_processor()
+
+        # The chat template is already applied to the prompt tokens
+        # Use message_format="none" to avoid applying it again
+        # Prepend an empty space if `always_start_with_space` is True
+        tokens = processor.get_tokens_input(
+            self.info.get_tokenizer().decode(prompt),
+            message_format="none",
+            always_start_with_space=True,
+        )
+
+        # Prepend a BOS token id to the tokens
+        return self.info.ctx.call_hf_processor(
+            processor.process,
+            dict(tokens=tokens),
+        )["input_ids"].tolist()
+
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        hf_processor = self.info.get_hf_processor(**mm_kwargs)
-        processed_outputs = self.info.ctx.call_hf_processor(
+        mm_counts = mm_items.get_all_counts()
+
+        valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
+        processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not processor_data:
+            return BatchFeature(dict(passthrough_data))
+
+        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
+
+        processed_data = self.info.ctx.call_hf_processor(
             hf_processor.process,
-            dict(text=prompt, **mm_data),
-            mm_kwargs,
+            dict(text=prompt_text, **processor_data),
+            hf_processor_mm_kwargs,
         )
 
         tokenizer = hf_processor.tokenizer
@@ -1141,17 +1168,16 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
 
         image_processor = hf_processor.image_processor
 
-        input_ids: torch.Tensor = processed_outputs.pop("input_ids")
-        processed_outputs["input_ids"] = input_ids.unsqueeze(0)
+        processed_data.pop("input_ids")
 
-        if (images := mm_data.get("images")) is not None:
+        if (images := processor_data.get("images")) is not None:
             mm_items = self.info.parse_mm_data({"image": images}, validate=False)
             parsed_images = mm_items.get_items("image", ImageProcessorItems)
             image_sizes = [
                 parsed_images.get_image_size(i) for i in range(len(parsed_images))
             ]
 
-            feat_is_patch = processed_outputs["image_input_idx"] >= 0
+            feat_is_patch = processed_data["image_input_idx"] >= 0
 
             tilings = [
                 self.info.select_tiling(
@@ -1165,35 +1191,12 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
             num_crops = torch.tensor(tilings).prod(-1) + 1
             assert num_crops.sum() == len(feat_is_patch)
 
-            processed_outputs["num_crops"] = num_crops
-            processed_outputs["img_patch_id"] = image_patch_id
+            processed_data["num_crops"] = num_crops
+            processed_data["img_patch_id"] = image_patch_id
 
-        return processed_outputs
+        processed_data.update(passthrough_data)
 
-    def _apply_hf_processor_tokens_only(
-        self,
-        prompt_tokens: list[int],
-    ) -> list[int]:
-        processor = self.info.get_hf_processor()
-
-        # The chat template is already applied to the prompt tokens
-        # Use message_format="none" to avoid applying it again
-        # Prepend an empty space if `always_start_with_space` is True
-        tokens = processor.get_tokens_input(
-            self.info.get_tokenizer().decode(prompt_tokens),
-            message_format="none",
-            always_start_with_space=True,
-        )
-
-        # Prepend a BOS token id to the tokens
-        processed_data = self.info.ctx.call_hf_processor(
-            processor.process,
-            dict(tokens=tokens),
-        )
-        prompt_ids = processed_data.pop("input_ids").tolist()
-        print(prompt_ids, len(prompt_ids))
-
-        return prompt_ids
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/molmo2.py

@@ -1929,30 +1929,33 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
 
 
 class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
-    def _apply_hf_processor_tokens_only(
-        self,
-        prompt_tokens: list[int],
-    ) -> list[int]:
+    def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
         processor = self.info.get_hf_processor()
         tokenizer = processor.tokenizer
         bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
 
-        if len(prompt_tokens) == 0 or prompt_tokens[0] != bos_token_id:
+        if len(prompt) == 0 or prompt[0] != bos_token_id:
             # Prepend the bos token to the prompt tokens
-            prompt_tokens = [bos_token_id] + prompt_tokens
+            prompt = [bos_token_id] + prompt
 
-        return prompt_tokens
+        return prompt
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        mm_data = dict(mm_data)
+        mm_counts = mm_items.get_all_counts()
+
+        valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
+        processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
+
+        mm_data = dict(processor_data)
 
         hf_config = self.info.get_hf_config()
-        hf_processor = self.info.get_hf_processor(**mm_kwargs)
+        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
 
         def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature:
             res = hf_processor(text=text, images=images, videos=videos, **kwargs)
@@ -1984,14 +1987,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
                 # NOTE: metadata.frames_indices indicates
                 # the sampled frames indices of pre-sampled videos, which is
                 # used to calculate the timestamps. Make sure that
-                # do_sample_frames in mm_kwargs is false for presampled videos.
+                # do_sample_frames in hf_processor_mm_kwargs is false for
+                # presampled videos.
 
-                # NOTE: a copy of mm_kwargs is created to update do_sample_frames,
-                # otherwise mm_hash for the object will be incorrect.
-                video_mm_kwargs = dict(**mm_kwargs)
+                # NOTE: a copy of hf_processor_mm_kwargs is created to update
+                # do_sample_frames, otherwise mm_hash for the object will be
+                # incorrect.
+                video_mm_kwargs = dict(**hf_processor_mm_kwargs)
                 if "do_sample_frames" not in video_mm_kwargs:
                     # molmo_utils already has "do_sample_frames" in
-                    # mm_kwargs, don't overwrite it.
+                    # hf_processor_mm_kwargs, don't overwrite it.
                     video_mm_kwargs["do_sample_frames"] = metadata.get(
                         "do_sample_frames", False
                     )
@@ -2015,7 +2020,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
                     input_ids = input_ids[:, 1:]
 
                 video_string = tokenizer.batch_decode(input_ids)[0]
-                prompt = prompt.replace(VIDEO_PROMPT, video_string, 1)
+                prompt_text = prompt_text.replace(VIDEO_PROMPT, video_string, 1)
 
                 video_grids = video_outputs.pop("video_grids")
                 assert video_grids[:, 0].sum() == len(
@@ -2054,10 +2059,10 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
         else:
             all_video_outputs = dict()
 
-        processed_outputs = self.info.ctx.call_hf_processor(
+        processed_data = self.info.ctx.call_hf_processor(
             patched_call,
-            dict(text=prompt, **mm_data),
-            mm_kwargs,
+            dict(text=prompt_text, **mm_data),
+            hf_processor_mm_kwargs,
         )
 
         if (images := mm_data.get("images")) is not None:
@@ -2077,22 +2082,22 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
                 for image_size in image_sizes
             ]
             num_crops = torch.tensor(tilings).prod(-1) + 1
-            assert sum(num_crops) == len(processed_outputs["pixel_values"])
-            assert sum(num_crops) == processed_outputs["image_num_crops"].sum().item()
+            assert sum(num_crops) == len(processed_data["pixel_values"])
+            assert sum(num_crops) == processed_data["image_num_crops"].sum().item()
 
-            image_grids = processed_outputs.pop("image_grids")
+            image_grids = processed_data.pop("image_grids")
             image_num_pooled_patches = image_grids[:, :2].prod(dim=1) + image_grids[
                 :, 2:
             ].prod(dim=1)
 
-            processed_outputs["image_num_pooled_patches"] = image_num_pooled_patches
-            n_patches = processed_outputs["pixel_values"].shape[1]
-            processed_outputs["image_num_patches"] = (
-                processed_outputs["image_num_crops"] * n_patches
+            processed_data["image_num_pooled_patches"] = image_num_pooled_patches
+            n_patches = processed_data["pixel_values"].shape[1]
+            processed_data["image_num_patches"] = (
+                processed_data["image_num_crops"] * n_patches
             )
             (
-                processed_outputs["image_tokens"],
-                processed_outputs["num_image_tokens"],
+                processed_data["image_tokens"],
+                processed_data["num_image_tokens"],
             ) = build_flat_image_bool_length(
                 image_grids,
                 hf_config,
@@ -2101,7 +2106,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
                 use_single_crop_start_token=hf_processor.use_single_crop_start_token,
             )
 
-        return BatchFeature({**processed_outputs, **all_video_outputs})
+        processed_data.update(all_video_outputs)
+        processed_data.update(passthrough_data)
+        processed_data.pop("input_ids")
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/moondream3.py

@@ -942,16 +942,6 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
     image_placeholder: str = "<image>"
     bos_image_placeholder: str = "<|endoftext|><image>"
 
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        # Moondream3's processor handles images directly rather than exposing a
-        # separate `image_processor`, so keep the cache path on text+MM calls.
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
-
     @cached_property
     def bos_image_placeholder_tokens(self) -> list[int]:
         tokenizer = self.info.get_tokenizer()

vllm/model_executor/models/moss_audio.py

@@ -1335,23 +1335,32 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo
 
 
 class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
         mm_data = dict(mm_data)
         audios = mm_data.pop("audios", [])
         if audios:
             mm_data["audio"] = audios
-        mm_kwargs = dict(mm_kwargs)
-        processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs)
-        return self.info.ctx.call_hf_processor(
+        hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
+        processor_kwargs = _filter_moss_audio_processor_config(hf_processor_mm_kwargs)
+        processed_data = self.info.ctx.call_hf_processor(
             self.info.get_hf_processor(**processor_kwargs),
-            dict(text=prompt, **mm_data),
+            mm_data,
             {},
         )
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/moss_transcribe_diarize.py

@@ -461,24 +461,29 @@ class MossTranscribeDiarizeDummyInputsBuilder(
 class MossTranscribeDiarizeMultiModalProcessor(
     BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]
 ):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        tokenizer = self.info.get_tokenizer()
-        audios = _get_audios_from_mm_data(mm_data)
-        if not audios:
-            input_ids = tokenizer.encode(prompt, add_special_tokens=False)
-            return BatchFeature({"input_ids": [input_ids]}, tensor_type="pt")
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
 
-        processed = self.info.ctx.call_hf_processor(
-            self.info.get_hf_processor(**mm_kwargs),
-            dict(text=prompt, audio=audios),
-            mm_kwargs,
-        )
-        return _add_vllm_audio_metadata(processed, len(audios))
+    def _preprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        audios = _get_audios_from_mm_data(mm_data)
+
+        return dict(audio=audios), hf_processor_mm_kwargs
+
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        audios = _get_audios_from_mm_data(mm_data)
+        if audios:
+            processed_data = _add_vllm_audio_metadata(processed_data, len(audios))
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/muse_glimmer.py

@@ -262,13 +262,19 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
 class MuseGlimmerMultiModalProcessor(
     BaseMultiModalProcessor[MuseGlimmerProcessingInfo]
 ):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        processor = self.info.get_hf_processor(**mm_kwargs)
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
+        processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
         tokenizer = processor.tokenizer
         config = self.info.get_hf_config()
         images = mm_data.get("images", ())
@@ -276,7 +282,7 @@ class MuseGlimmerMultiModalProcessor(
         if not isinstance(images, Sequence) or not isinstance(videos, Sequence):
             raise TypeError("MuseGlimmer multi-modal data must be a sequence")
 
-        prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
+        prompt_ids = tokenizer.encode(prompt_text, add_special_tokens=False)
         image_sentinel_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)
         video_sentinel_id = tokenizer.convert_tokens_to_ids(VIDEO_TOKEN)
         if prompt_ids.count(image_sentinel_id) != len(images):
@@ -354,7 +360,9 @@ class MuseGlimmerMultiModalProcessor(
                 video_pixel_values=video_pixels,
                 video_feature_sizes=torch.tensor(video_sizes),
             )
-        return BatchFeature(data=data, tensor_type=None)
+        processed_data = BatchFeature(data=data, tensor_type=None)
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/nano_nemotron_vl.py

@@ -361,18 +361,8 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
 class NanoNemotronVLMultiModalProcessor(
     BaseMultiModalProcessor[NanoNemotronVLProcessingInfo]
 ):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        """
-        Bypass `call_hf_processor_mm_only` by no-op overriding`_call_hf_processor`,
-        so it chooses this path:
-        `type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor`
-        """
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
 
     def _get_image_fields_config(self, hf_inputs: BatchFeature):
         if self.info.is_dynamic_tiler:
@@ -406,7 +396,9 @@ class NanoNemotronVLMultiModalProcessor(
         )
 
     def _get_audio_fields_config(self, hf_inputs: BatchFeature):
-        audio_num_clips = torch.as_tensor(hf_inputs["audio_num_clips"])
+        audio_num_clips = torch.as_tensor(
+            hf_inputs.get("audio_num_clips", torch.empty(0))
+        )
 
         return dict(
             input_audio_features=MultiModalFieldConfig.flat_from_sizes(
@@ -733,11 +725,8 @@ class NanoNemotronVLMultiModalProcessor(
         # Bypass the cached path: the HF processor must receive the
         # prompt (with injected <so_embedding>) and the audio data
         # together so it can perform audio-token replacement natively.
-        (
-            prompt_ids,
-            mm_info,
-            is_update_applied,
-        ) = self._apply_hf_processor(inputs, timing_ctx)
+        mm_info = self._apply_hf_processor(inputs, timing_ctx)
+        prompt_ids = self._postprocess_prompt(inputs.prompt)
 
         with timing_ctx.record("apply_prompt_updates"):
             prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
@@ -745,7 +734,7 @@ class NanoNemotronVLMultiModalProcessor(
                 prompt_ids=prompt_ids,
                 mm_kwargs=mm_info.kwargs,
                 mm_prompt_updates=mm_info.prompt_updates,
-                is_update_applied=is_update_applied,
+                is_update_applied=self.hf_processor_applies_updates,
             )
 
         mm_placeholder_ranges = {

vllm/model_executor/models/nemotron_parse.py

@@ -411,22 +411,6 @@ class NemotronParseMultiModalProcessor(
     ) -> list[int]:
         return [0]
 
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        if mm_data:
-            processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
-        else:
-            hf_processor = self.info.get_hf_processor()
-            tokenizer = hf_processor.tokenizer
-            processed_outputs = tokenizer(
-                prompt, add_special_tokens=False, return_tensors="pt"
-            )
-        return processed_outputs
-
     def _get_mm_fields_config(
         self,
         hf_inputs: BatchFeature,

vllm/model_executor/models/ovis.py

@@ -341,41 +341,30 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
         # -300 is image_atom token, filter them out
         return [vte_vocab_size + x + 300 for x in image_indicators if x < -300]
 
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _postprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
     ) -> BatchFeature:
         if not mm_data:
-            # Avoid warning from HF logger for text-only input
-            tokenizer = self.info.get_tokenizer()
-            prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
-
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
+            return processed_data
 
         hf_processor = self.info.get_hf_processor()
         image_indicators = [
             hf_processor.construct_image_indicators(grid)
-            for grid in processed_outputs["grids"]
+            for grid in processed_data["grids"]
         ]
         indicator_tokens = [
             self.image_indicators_to_visual_tokens(indicator)
             for indicator in image_indicators
         ]
-        processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens)
-        return processed_outputs
+        processed_data["indicator_tokens"] = torch.tensor(indicator_tokens)
 
-    def _apply_hf_processor_tokens_only(
-        self,
-        prompt_tokens: list[int],
-    ) -> list[int]:
-        return prompt_tokens
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/ovis2_5.py

@@ -332,53 +332,43 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
             if x >= INDICATOR_IDS[0]
         ]
 
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _postprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
     ) -> BatchFeature:
         if not mm_data:
-            # Avoid warning from HF logger for text-only input
-            tokenizer = self.info.get_tokenizer()
-            prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
+            return processed_data
 
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
         hf_processor = self.info.get_hf_processor()
 
         if "videos" in mm_data:
             visual_indicators = [
                 hf_processor.construct_visual_indicators((1, 1, 1), True)
-                for grid in processed_outputs["video_grids"]
+                for grid in processed_data["video_grids"]
             ]
             indicator_tokens = [
                 self.visual_indicators_to_visual_tokens(indicator)
                 for indicator in visual_indicators
             ]
-            processed_outputs["video_indicator_tokens"] = torch.tensor(indicator_tokens)
+            processed_data["video_indicator_tokens"] = torch.tensor(indicator_tokens)
         if "images" in mm_data:
             visual_indicators = [
                 hf_processor.construct_visual_indicators((1, 1, 1), False)
-                for grid in processed_outputs["grids"]
+                for grid in processed_data["grids"]
             ]
             indicator_tokens = [
                 self.visual_indicators_to_visual_tokens(indicator)
                 for indicator in visual_indicators
             ]
 
-            processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens)
-        return processed_outputs
+            processed_data["indicator_tokens"] = torch.tensor(indicator_tokens)
 
-    def _apply_hf_processor_tokens_only(
-        self,
-        prompt_tokens: list[int],
-    ) -> list[int]:
-        return prompt_tokens
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/paddleocr_vl.py

@@ -244,32 +244,37 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing
 class PaddleOCRVLMultiModalProcessor(
     BaseMultiModalProcessor[PaddleOCRVLProcessingInfo]
 ):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        if mm_data:
-            final_mm_kwargs = dict(mm_kwargs or {})
-            final_mm_kwargs.setdefault("images_kwargs", {})
-            # vLLM use PIL.Image, always set channel_last
-            final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
-            processed_outputs = self.info.ctx.call_hf_processor(
-                self.info.get_hf_processor(**final_mm_kwargs),
-                dict(text=prompt, **mm_data),
-                mm_kwargs,
-            )
-            num_patches_per_image = processed_outputs["image_grid_thw"].prod(-1)
-            processed_outputs["pixel_values"] = processed_outputs["pixel_values"].split(
-                num_patches_per_image.tolist()
-            )
-        else:
-            tokenizer = self.info.get_tokenizer()
-            processed_outputs = tokenizer(
-                prompt, add_special_tokens=True, return_tensors="pt"
-            )
-        return processed_outputs
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
+        final_mm_kwargs = dict(hf_processor_mm_kwargs or {})
+        final_mm_kwargs.setdefault("images_kwargs", {})
+        # vLLM use PIL.Image, always set channel_last
+        final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
+        processed_data = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**final_mm_kwargs),
+            dict(text=prompt_text, **mm_data),
+            hf_processor_mm_kwargs,
+        )
+        num_patches_per_image = processed_data["image_grid_thw"].prod(-1)
+        processed_data["pixel_values"] = processed_data["pixel_values"].split(
+            num_patches_per_image.tolist()
+        )
+        processed_data.update(passthrough_data)
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/paligemma.py

@@ -152,23 +152,6 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo
 
 
 class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingInfo]):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        tokenizer = self.info.get_tokenizer()
-        if not mm_data:
-            prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
-
-        return super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
-
     def _get_mm_fields_config(
         self,
         hf_inputs: BatchFeature,

vllm/model_executor/models/phi3v.py

@@ -396,19 +396,19 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
 
 
 class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
 
-        input_ids = processed_outputs["input_ids"]
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if not mm_data:
+            return processed_data
+
+        input_ids = processed_data["input_ids"]
         assert isinstance(input_ids, torch.Tensor)
 
         # Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
@@ -416,7 +416,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
         # Therefore, we need to do an early replacement here
         input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID)
 
-        return processed_outputs
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/phi4mm.py

@@ -868,43 +868,52 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
 
 
 class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        if not mm_data:
-            prompt_ids = self.info.get_tokenizer().encode(prompt)
-            prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
 
-        sr = self.info.get_feature_extractor(**mm_kwargs).sampling_rate
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
+        sr = self.info.get_feature_extractor(**hf_processor_mm_kwargs).sampling_rate
         if audio_data := mm_data.get("audios", []):
             mm_data["audios"] = [(data, sr) for data in audio_data]
 
-        processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
+        processed_data = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**hf_processor_mm_kwargs),
+            dict(text=prompt_text, **mm_data),
+            hf_processor_mm_kwargs,
+        )
 
-        hf_processor = self.info.get_hf_processor(**mm_kwargs)
+        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
         num_img_tokens = [
             self.info.get_num_image_tokens(
                 image_width=img_size[0],
                 image_height=img_size[1],
                 processor=hf_processor,
             )
-            for img_size in processed_outputs["image_sizes"]
+            for img_size in processed_data["image_sizes"]
         ]
-        processed_outputs["num_img_tokens"] = num_img_tokens
+        processed_data["num_img_tokens"] = num_img_tokens
 
-        audio_features = processed_outputs["input_audio_embeds"]
+        audio_features = processed_data["input_audio_embeds"]
         feature_sizes = [
             self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data
         ]
-        processed_outputs["input_audio_embeds"] = [
+        processed_data["input_audio_embeds"] = [
             audio_features[idx, :size] for idx, size in enumerate(feature_sizes)
         ]
+        processed_data.update(passthrough_data)
 
-        return processed_outputs
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/phi4siglip.py

@@ -142,16 +142,22 @@ class Phi4SiglipDummyInputsBuilder(
 class Phi4SiglipMultiModalProcessor(
     BaseMultiModalProcessor[Phi4SiglipProcessingInfo],
 ):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        processed = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
+        processed = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**hf_processor_mm_kwargs),
+            dict(text=prompt_text, **mm_data),
+            hf_processor_mm_kwargs,
         )
 
         # The HF processor's tokenizer_image_token() replaces the "<image>"
@@ -163,10 +169,12 @@ class Phi4SiglipMultiModalProcessor(
         # NOTE: tokenizer.__call__() (not .encode()) must be used so that
         # added/special tokens like <|user|>, <|end|> are kept as single IDs.
         tokenizer = self.info.get_tokenizer()
-        new_ids = tokenizer(prompt).input_ids
+        new_ids = tokenizer(prompt_text).input_ids
         processed["input_ids"] = torch.tensor([new_ids])
 
-        return processed
+        processed_data = processed
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/pixtral.py

@@ -51,12 +51,10 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
 from vllm.multimodal.processing.processor import (
     BaseMultiModalProcessor,
     BaseProcessingInfo,
-    MultiModalProcessingInfo,
     ProcessorInputs,
     PromptReplacement,
     PromptUpdate,
     PromptUpdateDetails,
-    TimingContext,
 )
 from vllm.platforms import current_platform
 from vllm.sequence import IntermediateTensors
@@ -213,6 +211,9 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
 
 
 class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]):
+    # The tokens are already inserted by the chat template
+    hf_processor_applies_updates = True
+
     def _get_mm_fields_config(
         self,
         hf_inputs: Mapping[str, NestedTensors],
@@ -220,24 +221,35 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
     ) -> Mapping[str, MultiModalFieldConfig]:
         return dict(images=MultiModalFieldConfig.batched("image"))
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
         outputs = self.info.ctx.call_hf_processor(
-            self.info.get_hf_processor(**mm_kwargs),
-            dict(text=prompt, **mm_data),
+            self.info.get_hf_processor(**hf_processor_mm_kwargs),
+            dict(text=prompt_text, **mm_data),
             # Avoid padding issue
-            dict(**mm_kwargs, return_tensors=None),
+            dict(**hf_processor_mm_kwargs, return_tensors=None),
         )
 
         # Missing batch dimension
         if is_list_of(outputs["input_ids"], int):
             outputs["input_ids"] = [outputs["input_ids"]]
 
-        return outputs
+        processed_data = outputs
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_prompt_updates(
         self,
@@ -273,16 +285,6 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
             ),
         ]
 
-    def _cached_apply_hf_processor(
-        self,
-        inputs: ProcessorInputs,
-        timing_ctx: TimingContext,
-    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
-        prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx)
-
-        # NOTE: The tokens are already inserted by the chat template
-        return prompt_ids, mm_info, True
-
 
 @MULTIMODAL_REGISTRY.register_processor(
     PixtralMultiModalProcessor,

vllm/model_executor/models/qwen2_5_omni_thinker.py

@@ -499,83 +499,6 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
 class Qwen2_5OmniThinkerMultiModalProcessor(
     BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
 ):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        mm_data = dict(mm_data)
-        audios = mm_data.pop("audios", [])
-
-        # NOTE: WhisperFeatureExtractor cannot handle empty list of audios
-        if audios:
-            # NOTE: Qwen2.5-Omni processor accept "audio"
-            mm_data["audio"] = audios
-            mm_kwargs = dict(
-                **mm_kwargs,
-            )
-        elif (
-            mm_kwargs.get("use_audio_in_video")
-            and "audio" not in mm_data
-            and mm_data.get("videos")
-        ):
-            # A subclass (e.g. Qwen3-Omni) may have already popped "audios"
-            # and populated mm_data["audio"] itself before delegating here,
-            # in which case this isn't a genuine "no audio" case. Likewise,
-            # mm_data can be empty on a multimodal-processor-cache hit, where
-            # there's nothing to (re-)process this call and the real result
-            # comes from the cache — not a genuine "no audio" case either.
-            raise ValueError(
-                "Video doesn't have audio track with `audio_in_video=True`"
-            )
-
-        merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
-        if mm_data.get("videos") and (
-            merged.keys() & {"size", "min_pixels", "max_pixels"}
-        ):
-            mm_kwargs = dict(mm_kwargs)
-            video_size = dict(self.info.get_hf_processor().video_processor.size)
-            size_override = merged.get("size")
-            if size_override is not None:
-                video_size = video_size | size_override
-            min_pixels = merged.get("min_pixels")
-            if min_pixels is not None:
-                video_size["shortest_edge"] = min_pixels
-            max_pixels = merged.get("max_pixels")
-            if max_pixels is not None:
-                video_size["longest_edge"] = max_pixels
-            mm_kwargs["size"] = video_size
-
-        hf_inputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
-
-        input_features = hf_inputs.pop("input_features", None)
-        feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
-        if "input_audio_features" not in hf_inputs and input_features is not None:
-            if feature_attention_mask is not None:
-                input_features = input_features.permute(0, 2, 1)[
-                    feature_attention_mask.bool()
-                ].permute(1, 0)
-            hf_inputs["input_audio_features"] = input_features
-        if (
-            "audio_feature_lengths" not in hf_inputs
-            and feature_attention_mask is not None
-        ):
-            hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
-
-        video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
-        if video_second_per_grid is not None:
-            hf_inputs["second_per_grid_ts"] = video_second_per_grid
-
-        use_audio_in_video = mm_kwargs.get("use_audio_in_video", False)
-        hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
-
-        return hf_inputs
-
     def _get_mm_fields_config(
         self,
         hf_inputs: BatchFeature,
@@ -885,7 +808,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
             ),
         ]
 
-    def _apply_hf_processor_mm_only(
+    def _apply_hf_processor_main(
         self,
         mm_items: MultiModalDataItems,
         hf_processor_mm_kwargs: Mapping[str, object],
@@ -903,13 +826,87 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
                 )
             mm_counts["audio"] -= mm_counts["video"]
 
-        _, mm_processed_data = self._apply_hf_processor_text_mm(
-            prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
-            mm_items=mm_items,
-            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
+
+        mm_data = dict(processor_data)
+        audios = mm_data.pop("audios", [])
+
+        # NOTE: WhisperFeatureExtractor cannot handle empty list of audios
+        if audios:
+            # NOTE: Qwen2.5-Omni processor accept "audio"
+            mm_data["audio"] = audios
+            hf_processor_mm_kwargs = dict(
+                **hf_processor_mm_kwargs,
+            )
+        elif (
+            hf_processor_mm_kwargs.get("use_audio_in_video")
+            and "audio" not in mm_data
+            and mm_data.get("videos")
+        ):
+            # A subclass (e.g. Qwen3-Omni) may have already popped "audios"
+            # and populated mm_data["audio"] itself before delegating here,
+            # in which case this isn't a genuine "no audio" case. Likewise,
+            # mm_data can be empty on a multimodal-processor-cache hit, where
+            # there's nothing to (re-)process this call and the real result
+            # comes from the cache — not a genuine "no audio" case either.
+            raise ValueError(
+                "Video doesn't have audio track with `audio_in_video=True`"
+            )
+
+        merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
+        if mm_data.get("videos") and (
+            merged.keys() & {"size", "min_pixels", "max_pixels"}
+        ):
+            hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
+            video_size = dict(self.info.get_hf_processor().video_processor.size)
+            size_override = merged.get("size")
+            if size_override is not None:
+                video_size = video_size | size_override
+            min_pixels = merged.get("min_pixels")
+            if min_pixels is not None:
+                video_size["shortest_edge"] = min_pixels
+            max_pixels = merged.get("max_pixels")
+            if max_pixels is not None:
+                video_size["longest_edge"] = max_pixels
+            hf_processor_mm_kwargs["size"] = video_size
+
+        hf_inputs = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**hf_processor_mm_kwargs),
+            dict(text=prompt_text, **mm_data),
+            hf_processor_mm_kwargs,
         )
 
-        return mm_processed_data
+        input_features = hf_inputs.pop("input_features", None)
+        feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
+        if "input_audio_features" not in hf_inputs and input_features is not None:
+            if feature_attention_mask is not None:
+                input_features = input_features.permute(0, 2, 1)[
+                    feature_attention_mask.bool()
+                ].permute(1, 0)
+            hf_inputs["input_audio_features"] = input_features
+        if (
+            "audio_feature_lengths" not in hf_inputs
+            and feature_attention_mask is not None
+        ):
+            hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
+
+        video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
+        if video_second_per_grid is not None:
+            hf_inputs["second_per_grid_ts"] = video_second_per_grid
+
+        use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
+        hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
+
+        processed_data = hf_inputs
+        processed_data.update(passthrough_data)
+        processed_data.pop("input_ids")
+
+        return processed_data
 
 
 class Qwen2_5OmniConditionalGenerationMixin:

vllm/model_executor/models/qwen2_5_vl.py

@@ -1154,16 +1154,6 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
             second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
         )
 
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        # Override to use the text path instead of token path to use the
-        # video-specific logic in processing_qwen2_5_vl.py
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
-
     def _get_prompt_updates(
         self,
         mm_items: MultiModalDataItems,

vllm/model_executor/models/qwen2_audio.py

@@ -235,36 +235,27 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
 
 
 class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, Any],
-    ) -> BatchFeature:
-        # NOTE - we rename audios -> audio in mm data because transformers has
-        # deprecated audios for the qwen2audio processor and will remove
-        # support for it in transformers 4.54.
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
+
+        mm_data = dict(mm_data)
         audios = mm_data.pop("audios", [])
         if audios:
             mm_data["audio"] = audios
 
-        # Text-only input not supported in composite processor
-        if not mm_data.get("audio", []):
-            prompt_ids = self.info.get_tokenizer().encode(prompt)
-            prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
-
-        feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
-        mm_kwargs = dict(
-            **mm_kwargs,
+        hf_processor_mm_kwargs = dict(
+            **hf_processor_mm_kwargs,
             sampling_rate=feature_extractor.sampling_rate,
         )
 
-        return super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
-        )
+        return mm_data, hf_processor_mm_kwargs
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/qwen3_omni_moe_thinker.py

@@ -1195,79 +1195,95 @@ Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder
 class Qwen3OmniMoeThinkerMultiModalProcessor(
     Qwen2_5OmniThinkerMultiModalProcessor,
 ):
-    def _call_hf_processor(
+    def _get_hf_mm_data(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        mm_data = dict(mm_data)
-        audios = mm_data.pop("audios", [])
+        mm_items: MultiModalDataItems,
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
 
-        def pad_to_hop_length(x: np.ndarray, hop_length: int) -> np.ndarray:
-            length = x.shape[-1]
-            if length % hop_length != 0:
-                pad_length = hop_length - (length % hop_length)
-                x = np.pad(x, (0, pad_length), mode="constant", constant_values=0)
-            return x
-
-        # NOTE: WhisperFeatureExtractor cannot handle empty list of audios
-        feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
-        hop_length = feature_extractor.hop_length
+        audios = processor_data.get("audios")
         if audios:
-            # NOTE: Qwen3-Omni processor accept "audio"
+            feature_extractor = self.info.get_feature_extractor()
+            hop_length = feature_extractor.hop_length
+
+            def pad_to_hop_length(x: np.ndarray) -> np.ndarray:
+                length = x.shape[-1]
+                if length % hop_length != 0:
+                    pad_length = hop_length - (length % hop_length)
+                    x = np.pad(x, (0, pad_length), mode="constant", constant_values=0)
+                return x
+
             # To make sure the cache works with padding=True, we pre-padded
-            # the audio to multiple of hop_length.
-            mm_data["audio"] = [
-                pad_to_hop_length(audio, hop_length)
+            # the audio to a multiple of hop_length.
+            processor_data = dict(processor_data)
+            processor_data["audios"] = [
+                pad_to_hop_length(audio)
                 if isinstance(audio, np.ndarray)
-                else (pad_to_hop_length(audio[0], hop_length), audio[1])
+                else (pad_to_hop_length(audio[0]), audio[1])
                 for audio in audios
             ]
 
+        return processor_data, passthrough_data
+
+    def _apply_hf_processor_main(
+        self,
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> BatchFeature:
+        if mm_items.get_all_counts().get("audio", 0):
             # TODO(Isotr0py): Remove this patch after upstream fix PR
             # released and Transformers version update:
             # https://github.com/huggingface/transformers/pull/41473
-            mm_kwargs = dict(mm_kwargs)
-            mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {})
-            mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {})
-        elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"):
-            # mm_data can be empty on a multimodal-processor-cache hit, where
-            # there's nothing to (re-)process this call and the real result
-            # comes from the cache — not a genuine "no audio" case.
-            raise ValueError(
-                "Video doesn't have audio track with `audio_in_video=True`"
+            hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
+            hf_processor_mm_kwargs["audio_kwargs"] = dict(
+                hf_processor_mm_kwargs.get("audio_kwargs") or {}
+            )
+            hf_processor_mm_kwargs["text_kwargs"] = dict(
+                hf_processor_mm_kwargs.get("text_kwargs") or {}
             )
 
-        hf_inputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
+        processed_data = super()._apply_hf_processor_main(
+            mm_items,
+            hf_processor_mm_kwargs,
         )
 
         if (
-            "audio_feature_lengths" in hf_inputs
-            and "feature_attention_mask" in hf_inputs
-            and (audios := mm_data.get("audio", []))
+            "audio_feature_lengths" in processed_data
+            and "feature_attention_mask" in processed_data
         ):
-            audio_num_frames = []
-            for _, audio in enumerate(audios):
-                audio_length = len(audio[0]) if isinstance(audio, tuple) else len(audio)
-                num_frame = (
-                    (audio_length // hop_length)
-                    if audio_length % hop_length == 0
-                    else (audio_length // hop_length - 1)
+            valid_mm_items = mm_items.select(
+                {k for k, c in mm_items.get_all_counts().items() if c > 0}
+            )
+            processor_data, _ = self._get_hf_mm_data(valid_mm_items)
+            audios = processor_data.get("audios", [])
+            if audios:
+                feature_extractor = self.info.get_feature_extractor(
+                    **hf_processor_mm_kwargs
                 )
-                if mm_kwargs.get("truncation", False):
-                    num_frame = min(
-                        num_frame, feature_extractor.n_samples // hop_length
+                hop_length = feature_extractor.hop_length
+
+                audio_num_frames = []
+                for audio in audios:
+                    audio_length = (
+                        len(audio[0]) if isinstance(audio, tuple) else len(audio)
                     )
-                audio_num_frames.append(num_frame)
-            hf_inputs["feature_attention_mask"] = [
-                torch.ones(num_frame) for num_frame in audio_num_frames
-            ]
-            hf_inputs["audio_feature_lengths"] = torch.tensor(audio_num_frames)
-        return hf_inputs
+                    num_frame = (
+                        (audio_length // hop_length)
+                        if audio_length % hop_length == 0
+                        else (audio_length // hop_length - 1)
+                    )
+                    if hf_processor_mm_kwargs.get("truncation", False):
+                        num_frame = min(
+                            num_frame, feature_extractor.n_samples // hop_length
+                        )
+                    audio_num_frames.append(num_frame)
+
+                processed_data["feature_attention_mask"] = [
+                    torch.ones(num_frame) for num_frame in audio_num_frames
+                ]
+                processed_data["audio_feature_lengths"] = torch.tensor(audio_num_frames)
+
+        return processed_data
 
     def _maybe_apply_prompt_updates(
         self,

vllm/model_executor/models/qwen3_vl.py

@@ -1280,12 +1280,21 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
         proc_impl = getattr(type(hf_processor), "replace_video_token", None)
         return proc_impl is not None and proc_impl is not mixin_impl
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
         mm_data = dict(mm_data)
 
         # Separate video processing from image processing. Because the videos
@@ -1314,8 +1323,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
 
                 # NOTE: a copy of is created to update do_sample_frames,
                 # otherwise mm_hash for the object will be incorrect.
-                video_mm_kwargs = dict(**mm_kwargs)
-                merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
+                video_mm_kwargs = dict(**hf_processor_mm_kwargs)
+                merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
                 if merged.keys() & {"size", "min_pixels", "max_pixels"}:
                     video_size = dict(self.info.get_video_processor().size)
                     size_override = merged.get("size")
@@ -1365,10 +1374,13 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
                 if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
                     video_mm_kwargs["fps"] = None
 
-                video_outputs = super()._call_hf_processor(
-                    prompt="<|vision_start|><|video_pad|><|vision_end|>",
-                    mm_data=video_mm_data,
-                    mm_kwargs=video_mm_kwargs,
+                video_outputs = self.info.ctx.call_hf_processor(
+                    self.info.get_hf_processor(**video_mm_kwargs),
+                    dict(
+                        text="<|vision_start|><|video_pad|><|vision_end|>",
+                        **video_mm_data,
+                    ),
+                    video_mm_kwargs,
                 )
 
                 # Discard HF output input_ids — we use get_video_repl below
@@ -1424,12 +1436,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
         # fps/num_frames are video-only kwargs already consumed by the loop;
         # exclude them so the text/image processor call below never gets a list.
         non_video_mm_kwargs = {
-            k: v for k, v in mm_kwargs.items() if k not in ("fps", "num_frames")
+            k: v
+            for k, v in hf_processor_mm_kwargs.items()
+            if k not in ("fps", "num_frames")
         }
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=non_video_mm_kwargs,
+        processed_data = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**non_video_mm_kwargs),
+            dict(text=prompt_text, **mm_data),
+            non_video_mm_kwargs,
         )
 
         # Replace each placeholder with pre-computed video tokens.
@@ -1443,20 +1457,19 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
                     hf_config.video_token_id,
                     hf_config.vision_end_token_id,
                 ]
-            input_ids = processed_outputs.pop("input_ids")
+            input_ids = processed_data.pop("input_ids")
             if not isinstance(input_ids, list):
                 input_ids = input_ids.tolist()
             (prompt_ids,) = input_ids
             expanded_ids = _replace_video_token_placeholders(
                 prompt_ids, video_target, video_input_ids_lst
             )
-            processed_outputs["input_ids"] = [expanded_ids]
+            processed_data["input_ids"] = [expanded_ids]
 
-        combined_outputs = dict(
-            processed_outputs,
-            **video_outputs,
-        )
-        return BatchFeature(combined_outputs)
+        processed_data.update(video_outputs)
+        processed_data.update(passthrough_data)
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/transformers/multimodal.py

@@ -345,28 +345,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
 
         return mm_fields
 
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> "BatchFeature":
-        """Run the HF processor and unpad inputs."""
-        try:
-            hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
-        except ValueError:
-            if any(mm_data.values()):
-                raise
-            # Some processors reject a prompt holding placeholders with
-            # no data to go with them, so tokenize it without them
-            prompt_ids = self.info.get_tokenizer().encode(prompt)
-            hf_inputs = transformers.BatchFeature(
-                dict(input_ids=[prompt_ids]), tensor_type="pt"
-            )
-        self._unpad_images(hf_inputs)
-        self._unpad_audios(hf_inputs, mm_data, mm_kwargs)
-        return hf_inputs
-
     def _unpad_audios(
         self,
         hf_inputs: "BatchFeature",
@@ -450,19 +428,6 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
         deriving them from updates."""
         return []
 
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> "BatchFeature":
-        """Ask which modality owns each token of the expanded prompt, which is the
-        only marking of the tokens an expansion adds around an item."""
-        if any(mm_data.values()):
-            mm_data = {**mm_data, "return_mm_token_type_ids": True}
-
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
-
     def _get_mm_token_ids(self, modality: str) -> list[int]:
         """Token ids marking where `modality` sits in the prompt, which for some
         processors differ from the placeholder written into it.
@@ -612,24 +577,55 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
 
         with timing_ctx.record("apply_hf_processor"):
             hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
-            prompt = hf_processor.decode(prompt)
+            prompt_text = hf_processor.decode(prompt)
 
             # Bypass cached processor and always apply to the full set of mm inputs
             # NOTE: we can't just set caching=False because base class method
             # transforms outputs to `MultiModalKwargs` which is not going to
             # work for Transformers. The vision path has logic tied to
             # `mm_tokens_per_modality` in _apply_vision()
-            prompt_ids, processed_data = self._apply_hf_processor_text_mm(
-                prompt_text=prompt,
-                mm_items=mm_items,
-                hf_processor_mm_kwargs={
-                    **hf_processor_mm_kwargs,
-                    # HF processors only accept text, and the decoded string already
-                    # contains any special tokens, so don't let them be added again
-                    # (`_call_hf_processor` disables `add_special_tokens`).
-                    "add_special_tokens": False,
-                },
+            hf_processor_mm_kwargs = {
+                **hf_processor_mm_kwargs,
+                # HF processors only accept text, and the decoded string already
+                # contains any special tokens, so don't let them be added again
+                # (the HF processor call disables `add_special_tokens`).
+                "add_special_tokens": False,
+            }
+
+            valid_mm_items = mm_items.select(
+                {k for k, c in mm_items.get_all_counts().items() if c > 0}
             )
+            processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+            # Ask which modality owns each token of the expanded prompt, which
+            # is the only marking of the tokens an expansion adds around an item
+            if any(processor_data.values()):
+                processor_data = {**processor_data, "return_mm_token_type_ids": True}
+
+            try:
+                processed_data = self.info.ctx.call_hf_processor(
+                    self.info.get_hf_processor(**hf_processor_mm_kwargs),
+                    dict(text=prompt_text, **processor_data),
+                    hf_processor_mm_kwargs,
+                )
+            except ValueError:
+                if any(processor_data.values()):
+                    raise
+                # Some processors reject a prompt holding placeholders with
+                # no data to go with them, so tokenize it without them
+                prompt_ids = self.info.get_tokenizer().encode(prompt_text)
+                processed_data = transformers.BatchFeature(
+                    dict(input_ids=[prompt_ids]), tensor_type="pt"
+                )
+            self._unpad_images(processed_data)
+            self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs)
+            processed_data.update(passthrough_data)
+
+            input_ids = processed_data.pop("input_ids")
+            if not isinstance(input_ids, list):
+                input_ids = input_ids.tolist()
+
+            (prompt_ids,) = input_ids
 
         # Use overrides if provided; fallback to data-dependent hashing.
         with timing_ctx.record("get_mm_hashes"):
@@ -773,17 +769,41 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
         except (AttributeError, KeyError, TypeError):
             return None
 
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> "BatchFeature":
-        """Ask for the replacement each placeholder expands to, and record it as
-        per-item fields: its token ids, and the tokens or patches behind them."""
+        mm_counts = mm_items.get_all_counts()
+
+        valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
+
+        # Ask for the replacement each placeholder expands to, and record it as
+        # per-item fields: its token ids, and the tokens or patches behind them
         if has_mm_data := any(mm_data.values()):
             mm_data = {**mm_data, "return_text_replacement_offsets": True}
-        hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
+
+        try:
+            hf_inputs = self.info.ctx.call_hf_processor(
+                self.info.get_hf_processor(**hf_processor_mm_kwargs),
+                dict(text=prompt_text, **mm_data),
+                hf_processor_mm_kwargs,
+            )
+        except ValueError:
+            if any(mm_data.values()):
+                raise
+            # Some processors reject a prompt holding placeholders with
+            # no data to go with them, so tokenize it without them
+            tokenizer = self.info.get_tokenizer()
+            hf_inputs = transformers.BatchFeature(
+                dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
+            )
+        self._unpad_images(hf_inputs)
+        self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)
+
         # Drop the inputs the model would reject
         hf_inputs.pop("mm_token_type_ids", None)
         hf_inputs.pop("token_type_ids", None)
@@ -802,6 +822,8 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
                     "so it can be fixed, and install `transformers<5.15.0` in the "
                     "meantime to locate placeholders in the expanded prompt instead."
                 )
+            hf_inputs.update(passthrough_data)
+            hf_inputs.pop("input_ids", None)
             return hf_inputs
 
         tokenizer = self.info.get_tokenizer()
@@ -829,6 +851,9 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
                     counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
                 hf_inputs["num_audio_tokens"] = torch.tensor(counts)
 
+        hf_inputs.update(passthrough_data)
+        hf_inputs.pop("input_ids")
+
         return hf_inputs
 
 

vllm/model_executor/models/ultravox.py

@@ -190,41 +190,84 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
 
 
 class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        # Text-only input not supported in composite processor
-        if not mm_data.get("audios", []):
-            prompt_ids = self.info.get_tokenizer().encode(
-                prompt, add_special_tokens=False
-            )
-            prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
-            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
 
-        mm_data = dict(mm_data)
-        audios = mm_data.pop("audios", [])
+    def _preprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        audios = mm_data.get("audios", [])
         assert isinstance(audios, list)
 
-        feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
-        mm_kwargs = dict(
-            **mm_kwargs,
+        feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
+        hf_processor_mm_kwargs = dict(
+            **hf_processor_mm_kwargs,
             sampling_rate=feature_extractor.sampling_rate,
             include_audio_num_chunks=True,
         )
 
-        item_processor_data = dict(**mm_data, audios=audios)
+        return mm_data, hf_processor_mm_kwargs
 
-        output = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=item_processor_data,
-            mm_kwargs=mm_kwargs,
+    def _apply_hf_processor_main(
+        self,
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
         )
-        output["audio_features"] = output.pop("audio_values")
+        processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
 
-        return output
+        if processor_data:
+            processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
+                processor_data,
+                hf_processor_mm_kwargs,
+            )
+
+            prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
+            if prompt_text is not None:
+                processor_data = dict(text=prompt_text, **processor_data)
+
+            hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
+
+            def patched_call(**kwargs) -> BatchFeature:
+                # The remote UltravoxProcessor does not support the
+                # `truncation` kwarg injected by `call_hf_processor`: it
+                # passes `truncation` to its audio processor by itself while
+                # also forwarding `**kwargs` there, causing a duplicate
+                # keyword argument error. Its hardcoded `truncation=False`
+                # already matches vLLM's intended behavior.
+                kwargs.pop("truncation", None)
+
+                return hf_processor(**kwargs)
+
+            processed_data = self.info.ctx.call_hf_processor(
+                patched_call,
+                processor_data,
+                hf_processor_mm_kwargs,
+            )
+            processed_data.update(passthrough_data)
+        else:
+            processed_data = BatchFeature(dict(passthrough_data))
+
+        return self._postprocess_hf_mm_data(
+            processor_data,
+            hf_processor_mm_kwargs,
+            processed_data,
+        )
+
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if "audio_values" in processed_data:
+            processed_data["audio_features"] = processed_data.pop("audio_values")
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/model_executor/models/unlimited_ocr.py

@@ -225,7 +225,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):
         self,
         inputs: ProcessorInputs,
         timing_ctx: TimingContext,
-    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
+    ) -> MultiModalProcessingInfo:
         # The processor logic differs for single-image (crop) vs multi-image
         # (no crop) requests. The processing cache assumes per-item output is
         # invariant of how many images are passed per prompt, so we only cache

vllm/model_executor/models/voxtral.py

@@ -47,12 +47,10 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
 from vllm.multimodal.processing.processor import (
     BaseMultiModalProcessor,
     BaseProcessingInfo,
-    MultiModalProcessingInfo,
     PlaceholderFeaturesInfo,
     ProcessorInputs,
     PromptReplacement,
     PromptUpdate,
-    TimingContext,
 )
 from vllm.sequence import IntermediateTensors
 from vllm.tokenizers import cached_tokenizer_from_config
@@ -204,6 +202,9 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
 
 
 class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
+    # The tokens are already inserted by the chat template
+    hf_processor_applies_updates = True
+
     def _get_mm_fields_config(
         self,
         hf_inputs: Mapping[str, NestedTensors],
@@ -220,12 +221,14 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
         # skip validation here
         pass
 
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
         mm_data = dict(mm_data)
         audios = mm_data.pop("audios", [])
 
@@ -233,18 +236,23 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
             # MistralCommonVoxtralProcessor accepts "audio"
             mm_data["audio"] = audios
 
-        outputs = self.info.ctx.call_hf_processor(
-            self.info.get_hf_processor(**mm_kwargs),
-            dict(text=prompt, **mm_data),
-            # Avoid padding issue
-            dict(**mm_kwargs, return_tensors=None),
-        )
+        # Avoid padding issue
+        hf_processor_mm_kwargs = dict(**hf_processor_mm_kwargs, return_tensors=None)
 
+        return mm_data, hf_processor_mm_kwargs
+
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
         # Missing batch dimension
-        if is_list_of(outputs["input_ids"], int):
-            outputs["input_ids"] = [outputs["input_ids"]]
+        input_ids = processed_data.get("input_ids")
+        if is_list_of(input_ids, int):
+            processed_data["input_ids"] = [input_ids]
 
-        return outputs
+        return processed_data
 
     def _get_prompt_updates(
         self,
@@ -287,16 +295,6 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
             ),
         ]
 
-    def _cached_apply_hf_processor(
-        self,
-        inputs: ProcessorInputs,
-        timing_ctx: TimingContext,
-    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
-        prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx)
-
-        # NOTE: The tokens are already inserted by the chat template
-        return prompt_ids, mm_info, True
-
 
 @MULTIMODAL_REGISTRY.register_processor(
     VoxtralMultiModalProcessor,

vllm/model_executor/models/whisper.py

@@ -756,27 +756,36 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
         # for encoder profiling.
         return [0]
 
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        if mm_data:
-            feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
-            mm_data = dict(audio=mm_data.pop("audios"))
-            mm_kwargs = dict(
-                **mm_kwargs,
-                sampling_rate=feature_extractor.sampling_rate,
-            )
-        processed_outputs = super()._call_hf_processor(
-            prompt=prompt,
-            mm_data=mm_data,
-            mm_kwargs=mm_kwargs,
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
+
+        mm_data = dict(mm_data)
+        mm_data["audio"] = mm_data.pop("audios")
+
+        hf_processor_mm_kwargs = dict(
+            **hf_processor_mm_kwargs,
+            sampling_rate=feature_extractor.sampling_rate,
         )
-        if "labels" in processed_outputs:
-            processed_outputs["input_ids"] = processed_outputs.pop("labels")
-        return processed_outputs
+
+        return mm_data, hf_processor_mm_kwargs
+
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
+    ) -> BatchFeature:
+        if "labels" in processed_data:
+            processed_data["input_ids"] = processed_data.pop("labels")
+
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/models/dots3_note/common/processor.py

@@ -649,13 +649,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
         processor_data["videos"] = raw_videos
         return processor_data, passthrough_data
 
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
+        return self.dummy_inputs.get_dummy_text(mm_counts)
 
     def _get_mm_fields_config(
         self,

vllm/models/inkling/common/mm_preprocess.py

@@ -22,7 +22,7 @@ from vllm.multimodal.inputs import (
     MultiModalFieldConfig,
     MultiModalKwargsItems,
 )
-from vllm.multimodal.parse import MultiModalDataParser
+from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
 from vllm.multimodal.processing import (
     BaseDummyInputsBuilder,
     BaseMultiModalProcessor,
@@ -135,8 +135,8 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
         num_images = mm_counts.get("image", 0)
         num_audios = mm_counts.get("audio", 0)
         # Use spellings the renderer would emit; tokenization is bypassed in
-        # _call_hf_processor (we build input_ids directly), so the exact text
-        # only needs to be a stable per-item marker.
+        # _apply_hf_processor_main (we build input_ids directly), so the exact
+        # text only needs to be a stable per-item marker.
         return ("<|content_image|>" * num_images) + (
             "<|content_audio_input|>" * num_audios
         )
@@ -179,17 +179,19 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
 
 
 class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
-        # Inkling is not a standard HF processor (no fused text+mm call), so we run
-        # the vendored extractors ourselves and tokenize the text separately.
-        # The MM placeholders in `prompt` are expanded later by the prompt
-        # updates, so here we emit ONE placeholder id per media item.
-        processor = self.info.get_hf_processor(**mm_kwargs)
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
+        processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
         tokenizer = self.info.get_tokenizer()
 
         images = mm_data.get("images") or []
@@ -200,7 +202,7 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
             audios = list(cast(Iterable[Any], audios))
 
         prompt_ids = self._tokenize_with_placeholders(
-            prompt, tokenizer, len(images), len(audios)
+            prompt_text, tokenizer, len(images), len(audios)
         )
 
         data: dict[str, Any] = {"input_ids": [prompt_ids]}
@@ -232,7 +234,9 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
             data["input_audio_features"] = input_audio_features
             data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)
 
-        return BatchFeature(data=data, tensor_type=None)
+        processed_data = BatchFeature(data=data, tensor_type=None)
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _tokenize_with_placeholders(
         self,

vllm/models/kimi_k3/common/mm_preprocess.py

@@ -231,19 +231,6 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
 class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
     """Image-only multi-modal processor for Kimi-K3."""
 
-    def _call_hf_processor(
-        self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        # Override so the base always routes through the text+mm path
-        # (`KimiK3Processor.__call__`). Otherwise the mm-only fast path calls
-        # the checkpoint image processor directly with bare PIL images, but it
-        # requires `{"type": "image", "image": PIL}` media dicts that only our
-        # wrapper builds.
-        return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
-
     def _get_mm_fields_config(
         self,
         hf_inputs: BatchFeature,

vllm/models/minimax_m3/common/mm_preprocess.py

@@ -306,12 +306,21 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing
 class MiniMaxM3VLMultiModalProcessor(
     BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo]
 ):
-    def _call_hf_processor(
+    def _apply_hf_processor_main(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
     ) -> BatchFeature:
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+
+        if not mm_data:
+            return BatchFeature(dict(passthrough_data))
+
+        prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
+
         mm_data = dict(mm_data)
         # With ``video_needs_metadata=True`` each video arrives as a
         # ``(frames, metadata)`` tuple. Split the frames back out and forward the
@@ -340,15 +349,17 @@ class MiniMaxM3VLMultiModalProcessor(
 
         # Override the video processor's default do_resize=False (set for a
         # pre-resized pipeline) to True for vLLM's raw-frame inputs.
-        merged = dict(do_resize=True, **mm_kwargs)
-        data = dict(text=prompt, **mm_data)
+        merged = dict(do_resize=True, **hf_processor_mm_kwargs)
+        data = dict(text=prompt_text, **mm_data)
         if video_metadata is not None:
             data["video_metadata"] = video_metadata
-        return self.info.ctx.call_hf_processor(
-            self.info.get_hf_processor(**mm_kwargs),
+        processed_data = self.info.ctx.call_hf_processor(
+            self.info.get_hf_processor(**hf_processor_mm_kwargs),
             data,
             merged,
         )
+        processed_data.update(passthrough_data)
+        return processed_data
 
     def _get_mm_fields_config(
         self,

vllm/multimodal/processing/context.py

@@ -262,9 +262,6 @@ class InputProcessingContext:
         hf_processor: Callable[..., BatchFeature] | ProcessorMixin,
         data: Mapping[str, object],
         kwargs: Mapping[str, object] = {},
-        *,
-        num_tries: int = 1,
-        max_tries: int = 5,
     ) -> BatchFeature:
         """
         Call `hf_processor` on the prompt `data`
@@ -274,6 +271,12 @@ class InputProcessingContext:
 
         merged_kwargs = self.get_merged_mm_kwargs(kwargs)
 
+        # vLLM needs the full untruncated sequence to keep multi-modal
+        # placeholder tokens aligned; note that the text inputs in
+        # call_hf_processor are just dummy text, not the original prompt.
+        # The original prompt is already tokenized by the renderer.
+        merged_kwargs.setdefault("truncation", False)
+
         allowed_kwargs = get_allowed_kwarg_only_overrides(
             hf_processor,
             merged_kwargs,

vllm/multimodal/processing/dummy_inputs.py

@@ -91,10 +91,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
             # prompt tokens.
             dummy_prompt = []
         else:
-            dummy_prompt = tokenizer.encode(
-                dummy_text,
-                **self.info.default_tok_params.get_encode_kwargs(),
-            )
+            dummy_prompt = tokenizer.encode(dummy_text, truncation=False)
 
         return ProcessorInputs(
             prompt=dummy_prompt,

vllm/multimodal/processing/processor.py

@@ -5,9 +5,10 @@ from collections import defaultdict, deque
 from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from enum import Enum
-from functools import lru_cache, partial
+from functools import lru_cache
 from typing import (
     TYPE_CHECKING,
+    ClassVar,
     Generic,
     NamedTuple,
     Protocol,
@@ -28,7 +29,6 @@ from vllm.inputs import (
 )
 from vllm.logger import init_logger
 from vllm.tokenizers import TokenizerLike
-from vllm.transformers_utils.processor import call_hf_processor_mm_only
 from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
 
 from ..inputs import (
@@ -1177,6 +1177,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
     Not to be confused with `transformers.ProcessorMixin`.
     """
 
+    hf_processor_applies_updates: ClassVar[bool] = False
+    """
+    Only set to True if the tokenizer or chat template
+    (which are run before MM processing) inserts placeholder tokens.
+    """
+
     def __init__(
         self,
         info: _I,
@@ -1302,131 +1308,100 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
 
         return processor_data, passthrough_data
 
-    def _call_hf_processor(
+    def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None:
+        """
+        Get the text to pass to the HF processor alongside the multi-modal
+        data.
+
+        By default, no text is passed. If the HF processor requires that
+        text and multi-modal items correspond to each other, you should
+        override this method to return dummy text generated by
+        [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder].
+        """
+        return None
+
+    def _preprocess_hf_mm_data(
         self,
-        prompt: str,
-        # Not to be confused with `mm_data` in `self.apply`.
-        # This refers to the data to be passed to HF processor.
         mm_data: Mapping[str, object],
-        mm_kwargs: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> tuple[Mapping[str, object], Mapping[str, object]]:
+        """
+        Pre-process the multi-modal data and HF processor keyword arguments
+        before they are passed to the HF processor.
+
+        By default, both are returned as-is. If the HF processor expects the
+        multi-modal data under different keys than those provided by the
+        multi-modal items (e.g. `audio` instead of `audios`), or requires
+        additional keyword arguments (e.g. `sampling_rate`), you should
+        override this method.
+        """
+        return mm_data, hf_processor_mm_kwargs
+
+    def _postprocess_hf_mm_data(
+        self,
+        mm_data: Mapping[str, object],
+        hf_processor_mm_kwargs: Mapping[str, object],
+        processed_data: BatchFeature,
     ) -> BatchFeature:
         """
-        Call the HF processor on the prompt text and
-        associated multi-modal data.
+        Post-process the output of the HF processor.
+
+        By default, the output is returned as-is. If you need to modify the
+        output of the HF processor before it is converted into multi-modal
+        keyword arguments, you should override this method.
         """
-        return self.info.ctx.call_hf_processor(
-            self.info.get_hf_processor(**mm_kwargs),
-            dict(text=prompt, **mm_data),
-            mm_kwargs,
-        )
-
-    def _apply_hf_processor_text_mm(
-        self,
-        prompt_text: str,
-        mm_items: MultiModalDataItems,
-        hf_processor_mm_kwargs: Mapping[str, object],
-    ) -> tuple[list[int], BatchFeature]:
-        """
-        Apply the HF processor on the prompt text and multi-modal data
-        together.
-        """
-        valid_mm_items = mm_items.select(
-            {k for k, c in mm_items.get_all_counts().items() if c > 0}
-        )
-        processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
-
-        processed_data = self._call_hf_processor(
-            prompt=prompt_text,
-            mm_data=processor_data,
-            mm_kwargs=hf_processor_mm_kwargs,
-        )
-        processed_data.update(passthrough_data)
-
-        input_ids = processed_data.pop("input_ids")
-        if not isinstance(input_ids, list):
-            input_ids = input_ids.tolist()
-
-        (prompt_ids,) = input_ids
-
-        return prompt_ids, processed_data
-
-    def _apply_hf_processor_tokens_only(
-        self,
-        prompt_tokens: list[int],
-    ) -> list[int]:
-        """
-        Apply the HF processor on the prompt tokens only.
-
-        Most HF processors accept prompt text but not prompt tokens.
-        If the HF processor adds or removes tokens that are not related to
-        multi-modal data, you should override this method so it is consistent
-        with the tokenization of the corresponding text.
-        """
-        return prompt_tokens
-
-    def _apply_hf_processor_mm_only(
-        self,
-        mm_items: MultiModalDataItems,
-        hf_processor_mm_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        """
-        Apply the HF processor on the multi-modal data only.
-
-        Since HF processor requires that text and multi-modal items
-        correspond to each other, we generate dummy text using
-        [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
-        to go along with the multi-modal data.
-        """
-        # Custom logic based on text inputs
-        if type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor:
-            mm_counts = mm_items.get_all_counts()
-
-            _, mm_processed_data = self._apply_hf_processor_text_mm(
-                prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
-                mm_items=mm_items,
-                hf_processor_mm_kwargs=hf_processor_mm_kwargs,
-            )
-
-            return mm_processed_data
-
-        valid_mm_items = mm_items.select(
-            {k for k, c in mm_items.get_all_counts().items() if c > 0}
-        )
-        processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
-
-        processed_data = self.info.ctx.call_hf_processor(
-            partial(
-                call_hf_processor_mm_only,
-                self.info.get_hf_processor(**hf_processor_mm_kwargs),
-            ),
-            processor_data,
-            hf_processor_mm_kwargs,
-        )
-        processed_data.update(passthrough_data)
-
         return processed_data
 
     def _apply_hf_processor_main(
         self,
-        prompt: list[int],
         mm_items: MultiModalDataItems,
         hf_processor_mm_kwargs: Mapping[str, object],
-    ) -> tuple[list[int], BatchFeature, bool]:
+    ) -> BatchFeature:
         """
-        Apply the HF processor on the prompt tokens and multi-modal data.
-
-        In addition, return whether prompt updates have been applied
-        (`False` here, since for token prompts the updates are normally
-        applied separately via `_maybe_apply_prompt_updates`).
+        Apply the HF processor on the multi-modal data.
         """
-        prompt_ids = self._apply_hf_processor_tokens_only(prompt)
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
 
-        mm_processed_data = self._apply_hf_processor_mm_only(
-            mm_items=mm_items,
-            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
+        if processor_data:
+            processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
+                processor_data, hf_processor_mm_kwargs
+            )
+
+            prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
+            if prompt_text is not None:
+                processor_data = dict(text=prompt_text, **processor_data)
+
+            processed_data = self.info.ctx.call_hf_processor(
+                self.info.get_hf_processor(**hf_processor_mm_kwargs),
+                processor_data,
+                hf_processor_mm_kwargs,
+            )
+            processed_data.update(passthrough_data)
+        else:
+            from transformers.feature_extraction_utils import BatchFeature
+
+            processed_data = BatchFeature(dict(passthrough_data))
+
+        return self._postprocess_hf_mm_data(
+            processor_data,
+            hf_processor_mm_kwargs,
+            processed_data,
         )
 
-        return prompt_ids, mm_processed_data, False
+    def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
+        """
+        Post-process the prompt token IDs before locating or applying
+        multi-modal prompt updates.
+
+        By default, the prompt is returned as-is. If the HF processor (or
+        chat template) applies additional transformations to the prompt
+        that are not reflected in its multi-modal outputs, you should
+        override this method to replicate them.
+        """
+        return prompt
 
     def _get_cache_missing_items(
         self,
@@ -1531,14 +1506,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
         self,
         inputs: ProcessorInputs,
         timing_ctx: TimingContext,
-    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
+    ) -> MultiModalProcessingInfo:
         with timing_ctx.record("apply_hf_processor"):
-            (
-                prompt_ids,
-                mm_processed_data,
-                is_update_applied,
-            ) = self._apply_hf_processor_main(
-                prompt=inputs.prompt,
+            mm_processed_data = self._apply_hf_processor_main(
                 mm_items=inputs.mm_data_items,
                 hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
             )
@@ -1569,13 +1539,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
             prompt_updates=mm_prompt_updates,
         )
 
-        return prompt_ids, mm_info, is_update_applied
+        return mm_info
 
     def _cached_apply_hf_processor(
         self,
         inputs: ProcessorInputs,
         timing_ctx: TimingContext,
-    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
+    ) -> MultiModalProcessingInfo:
         """
         Apply the HF processor on the full prompt text,
         caching the results and reusing cached results.
@@ -1599,16 +1569,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
                 mm_hashes=mm_hashes,
             )
 
-        # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
+        # NOTE: The prompt does not correspond to `mm_missing_data_items`,
         # so we can't apply prompt updates until the new multimodal
         # items are combined with the cached multimodal items
         with timing_ctx.record("apply_hf_processor"):
-            (
-                prompt_ids,
-                mm_missing_processed_data,
-                is_update_applied,
-            ) = self._apply_hf_processor_main(
-                prompt=inputs.prompt,
+            mm_missing_processed_data = self._apply_hf_processor_main(
                 mm_items=mm_missing_data_items,
                 hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
             )
@@ -1641,7 +1606,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
             prompt_updates=mm_prompt_updates,
         )
 
-        return prompt_ids, mm_info, is_update_applied
+        return mm_info
 
     def _apply_token_matches(
         self,
@@ -1770,7 +1735,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
                     "There is likely a problem with your "
                     "implementation of merged multi-modal processor for this "
                     "model (usually arising from an inconsistency between "
-                    "`_call_hf_processor` and `_get_mm_fields_config`)."
+                    "`_apply_hf_processor_main` and `_get_mm_fields_config`)."
                 )
 
     def _validate_mm_updates(
@@ -1805,8 +1770,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
                     f"Expected there to be {item_count} prompt placeholders "
                     f"corresponding to {item_count} {modality} items, but "
                     f"instead found {len(placeholders)} prompt placeholders! "
-                    "Make sure the implementation of `_call_hf_processor` and "
-                    "`_get_mm_fields_config` are consistent with each other."
+                    "Make sure the implementation of `_apply_hf_processor_main` "
+                    "and `_get_mm_fields_config` are consistent with each other."
                 )
 
     def _maybe_apply_prompt_updates(
@@ -1854,11 +1819,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
         3. Extract information about the placeholder tokens from the
            processed token IDs.
         """
-        (
-            prompt_ids,
-            mm_info,
-            is_update_applied,
-        ) = self._cached_apply_hf_processor(inputs, timing_ctx)
+        prompt_ids = self._postprocess_prompt(inputs.prompt)
+        mm_info = self._cached_apply_hf_processor(inputs, timing_ctx)
 
         with timing_ctx.record("apply_prompt_updates"):
             prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
@@ -1866,7 +1828,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
                 prompt_ids=prompt_ids,
                 mm_kwargs=mm_info.kwargs,
                 mm_prompt_updates=mm_info.prompt_updates,
-                is_update_applied=is_update_applied,
+                is_update_applied=self.hf_processor_applies_updates,
             )
 
         mm_placeholder_ranges = {

vllm/transformers_utils/processor.py

@@ -11,17 +11,13 @@ from transformers import (
     AutoImageProcessor,
     AutoProcessor,
     AutoVideoProcessor,
-    BatchFeature,
     processing_utils,
 )
-from transformers.audio_utils import AudioInput
 from transformers.feature_extraction_utils import FeatureExtractionMixin
 from transformers.image_processing_utils import BaseImageProcessor
-from transformers.image_utils import ImageInput
 from transformers.models.auto.video_processing_auto import VIDEO_PROCESSOR_MAPPING_NAMES
 from transformers.processing_utils import ProcessorMixin
 from transformers.video_processing_utils import BaseVideoProcessor
-from transformers.video_utils import VideoInput
 from typing_extensions import TypeVar
 
 from vllm.logger import init_logger
@@ -554,43 +550,3 @@ def cached_video_processor_from_config(
         processor_cls_overrides=processor_cls,  # type: ignore[arg-type]
         **_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs),
     )
-
-
-def call_hf_processor_mm_only(
-    processor: ProcessorMixin,
-    images: ImageInput | None = None,
-    videos: VideoInput | None = None,
-    audio: AudioInput | None = None,
-    **kwargs,
-) -> BatchFeature:
-    output_kwargs = processor._merge_kwargs(
-        get_processor_kwargs_type(processor),
-        **kwargs,
-    )
-
-    if audio is not None and (
-        feature_extractor := getattr(processor, "feature_extractor", None)
-    ):
-        audio_inputs = feature_extractor(audio, **output_kwargs["audio_kwargs"])
-        audio_inputs["feature_attention_mask"] = audio_inputs.pop("attention_mask")
-    else:
-        audio_inputs = {}
-
-    if images is not None and (
-        image_processor := getattr(processor, "image_processor", None)
-    ):
-        images_inputs = image_processor(images=images, **output_kwargs["images_kwargs"])
-    else:
-        images_inputs = {}
-
-    if videos is not None and (
-        video_processor := getattr(processor, "video_processor", None)
-    ):
-        videos_inputs = video_processor(videos=videos, **output_kwargs["videos_kwargs"])
-    else:
-        videos_inputs = {}
-
-    return BatchFeature(
-        data={**audio_inputs, **images_inputs, **videos_inputs},
-        tensor_type=kwargs.get("return_tensors"),
-    )