From 0b19ebcacd0e9578460206035c4b3575361c19c4 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sat, 22 Aug 2026 16:49:37 +0800 Subject: [PATCH] [MM] Simplify _apply_hf_processor_main (#53275) Signed-off-by: DarkLight1337 --- docs/contributing/model/multimodal.md | 69 +++-- docs/design/mm_processing.md | 53 +--- requirements/common.txt | 2 +- .../processing/test_audio_in_video.py | 2 +- .../processing/test_audioflamingo3.py | 4 +- .../multimodal/processing/test_common.py | 22 +- .../processing/test_llava_onevision2.py | 2 +- .../multimodal/processing/test_qwen3_vl.py | 4 +- vllm/model_executor/models/audioflamingo3.py | 47 ++-- vllm/model_executor/models/blip2.py | 18 -- vllm/model_executor/models/chameleon.py | 30 +-- vllm/model_executor/models/cheers.py | 9 - vllm/model_executor/models/cohere2_vision.py | 27 +- vllm/model_executor/models/cohere_asr.py | 65 +++-- vllm/model_executor/models/colmodernvbert.py | 23 +- vllm/model_executor/models/colpali.py | 40 +-- vllm/model_executor/models/deepseek_ocr.py | 29 ++- vllm/model_executor/models/deepseek_ocr2.py | 30 ++- vllm/model_executor/models/deepseek_vl2.py | 28 +-- vllm/model_executor/models/ernie45_vl.py | 39 +-- vllm/model_executor/models/fireredasr2.py | 45 ++-- vllm/model_executor/models/fireredlid.py | 45 ++-- vllm/model_executor/models/funasr.py | 62 +++-- vllm/model_executor/models/funaudiochat.py | 29 ++- vllm/model_executor/models/gemma3_mm.py | 25 +- vllm/model_executor/models/gemma3n_mm.py | 84 +++++-- vllm/model_executor/models/gemma4_mm.py | 66 ++--- vllm/model_executor/models/glm4_1v.py | 74 +++--- vllm/model_executor/models/glmasr.py | 76 +++--- vllm/model_executor/models/granite_speech.py | 28 ++- vllm/model_executor/models/h2ovl.py | 2 +- vllm/model_executor/models/hunyuan_vision.py | 22 -- .../models/hyperclovax_vision.py | 104 ++++---- .../models/hyperclovax_vision_v2.py | 28 ++- vllm/model_executor/models/idefics3.py | 45 ++-- vllm/model_executor/models/interns1.py | 76 ++++-- vllm/model_executor/models/internvl.py | 36 ++- vllm/model_executor/models/jina_vl.py | 28 ++- vllm/model_executor/models/kanana_v.py | 24 +- vllm/model_executor/models/keye.py | 11 +- vllm/model_executor/models/keye_vl1_5.py | 11 +- vllm/model_executor/models/kimi_audio.py | 26 +- vllm/model_executor/models/kimi_k25.py | 10 - vllm/model_executor/models/kimi_vl.py | 3 + vllm/model_executor/models/lfm2_vl.py | 33 +-- vllm/model_executor/models/lightonocr.py | 43 ++-- vllm/model_executor/models/llava.py | 30 +-- vllm/model_executor/models/llava_next.py | 3 + vllm/model_executor/models/llava_onevision.py | 55 ++-- .../model_executor/models/llava_onevision2.py | 81 +++--- vllm/model_executor/models/midashenglm.py | 29 +-- vllm/model_executor/models/mimo_v2_omni.py | 23 +- vllm/model_executor/models/minicpmo.py | 2 +- vllm/model_executor/models/minicpmv.py | 44 ++-- vllm/model_executor/models/mistral3.py | 30 +-- vllm/model_executor/models/mllama4.py | 32 +-- vllm/model_executor/models/molmo.py | 81 +++--- vllm/model_executor/models/molmo2.py | 73 +++--- vllm/model_executor/models/moondream3.py | 10 - vllm/model_executor/models/moss_audio.py | 25 +- .../models/moss_transcribe_diarize.py | 39 +-- vllm/model_executor/models/muse_glimmer.py | 22 +- .../model_executor/models/nano_nemotron_vl.py | 27 +- vllm/model_executor/models/nemotron_parse.py | 16 -- vllm/model_executor/models/ovis.py | 31 +-- vllm/model_executor/models/ovis2_5.py | 34 +-- vllm/model_executor/models/paddleocr_vl.py | 53 ++-- vllm/model_executor/models/paligemma.py | 17 -- vllm/model_executor/models/phi3v.py | 26 +- vllm/model_executor/models/phi4mm.py | 41 +-- vllm/model_executor/models/phi4siglip.py | 28 ++- vllm/model_executor/models/pixtral.py | 42 ++-- .../models/qwen2_5_omni_thinker.py | 163 ++++++------ vllm/model_executor/models/qwen2_5_vl.py | 10 - vllm/model_executor/models/qwen2_audio.py | 33 +-- .../models/qwen3_omni_moe_thinker.py | 124 +++++---- vllm/model_executor/models/qwen3_vl.py | 57 +++-- .../models/transformers/multimodal.py | 131 ++++++---- vllm/model_executor/models/ultravox.py | 93 +++++-- vllm/model_executor/models/unlimited_ocr.py | 2 +- vllm/model_executor/models/voxtral.py | 48 ++-- vllm/model_executor/models/whisper.py | 45 ++-- vllm/models/dots3_note/common/processor.py | 9 +- vllm/models/inkling/common/mm_preprocess.py | 32 +-- vllm/models/kimi_k3/common/mm_preprocess.py | 13 - .../models/minimax_m3/common/mm_preprocess.py | 27 +- vllm/multimodal/processing/context.py | 9 +- vllm/multimodal/processing/dummy_inputs.py | 5 +- vllm/multimodal/processing/processor.py | 236 ++++++++---------- vllm/transformers_utils/processor.py | 44 ---- 90 files changed, 1848 insertions(+), 1706 deletions(-) diff --git a/docs/contributing/model/multimodal.md b/docs/contributing/model/multimodal.md index 3d16e0e91f..36830dd8ea 100644 --- a/docs/contributing/model/multimodal.md +++ b/docs/contributing/model/multimodal.md @@ -153,11 +153,13 @@ def get_supported_mm_limits(self) -> Mapping[str, int | None]: ## 3. Specify dummy inputs -Then, inherit [BaseDummyInputsBuilder][vllm.multimodal.processing.BaseDummyInputsBuilder] to construct dummy inputs for -HF processing. The processed outputs are also used for memory profiling. +Then, inherit [BaseDummyInputsBuilder][vllm.multimodal.processing.BaseDummyInputsBuilder] to construct the dummy inputs +that are used for memory profiling. Override the abstract methods [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text] and [get_dummy_mm_data][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_mm_data] to construct dummy inputs. These dummy inputs should result in the worst-case memory usage of the model so that vLLM can reserve the correct amount of memory for it. +Besides profiling, the dummy text is passed to the HF processor by models whose HF processor requires text corresponding to the multi-modal items (see [Multi-modal fields](#multi-modal-fields)). If the HF processor can process multi-modal data without any text, the dummy text is not used during HF processing. + Assuming that the memory usage increases with the number of tokens, the dummy inputs can be constructed to maximize the number of output embeddings, which is the same number as placeholder feature tokens. === "Basic example: LLaVA" @@ -446,37 +448,64 @@ return a schema of the tensors outputted by the HF processor that are related to like in LLaVA, each image's features must be independent of the others (which is also required for prefix caching to work correctly). So, we un-pad each image back to its own size by overriding - [BaseMultiModalProcessor._call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor]: + [BaseMultiModalProcessor._postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data] + to post-process the outputs of the HF processor: ??? code ```python - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + # Mistral3Processor requires text corresponding to the images + return self.dummy_inputs.get_dummy_text(mm_counts) - pixel_values = processed_outputs.get("pixel_values") + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if not mm_data: + return processed_data + + pixel_values = processed_data.get("pixel_values") if pixel_values is not None: # Avoid padding since we need the output for each image to be # independent of other images for the cache to work correctly - image_sizes = processed_outputs["image_sizes"] + image_sizes = processed_data["image_sizes"] assert len(pixel_values) == len(image_sizes) - processed_outputs["pixel_values"] = [ + processed_data["pixel_values"] = [ p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes) ] - return processed_outputs + return processed_data ``` + The default implementation of + [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] + calls the HF processor on the multi-modal data without passing any text. + If the HF processor instead requires text corresponding to the multi-modal items, + you should override + [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text] + to return the dummy text from + [BaseDummyInputsBuilder.get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text] + like in the example above. If the HF processor expects the multi-modal data + under different keys than those provided by the multi-modal items + (e.g. `audio` instead of `audios`), or requires additional keyword arguments + (e.g. `sampling_rate`), you should override + [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data]. + If you need to modify the output of the HF processor, + you should override + [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data]. + If the HF processor applies additional transformations to the prompt tokens + that are not reflected in its multi-modal outputs (e.g. prepending a BOS token), + you should override + [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt] + to replicate them. + For even more control over how the HF processor is called, you can override + [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] + directly. + Since `pixel_values` is now a list with one tensor per image, we can override [_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows: @@ -680,7 +709,7 @@ Examples: ### Handling prompt updates unrelated to multi-modal data -[_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates] assumes that each application of prompt update corresponds to one multi-modal item. If the HF processor performs additional processing regardless of how many multi-modal items there are, you should override [_apply_hf_processor_tokens_only][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_tokens_only] so that the processed token inputs are consistent with the result of applying the HF processor on text inputs. This is because token inputs bypass the HF processor according to [our design](../../design/mm_processing.md). +[_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates] assumes that each application of prompt update corresponds to one multi-modal item. If the HF processor performs additional processing on the prompt regardless of how many multi-modal items there are, you should override [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt] so that the prompt token IDs are consistent with the result of applying the HF processor on text inputs. This is because token inputs bypass the HF processor according to [our design](../../design/mm_processing.md). Examples: @@ -690,7 +719,7 @@ Examples: ### Custom HF processor -Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and pass it to [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor]. +Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. It is then applied to the multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor] inside [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]. Examples: diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index b2b1ac307e..a2112a2922 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -2,7 +2,15 @@ To enable various optimizations in vLLM such as [chunked prefill](../configuration/optimization.md#chunked-prefill) and [prefix caching](../features/automatic_prefix_caching.md), we use [BaseMultiModalProcessor][vllm.multimodal.processing.BaseMultiModalProcessor] to provide the correspondence between placeholder feature tokens (e.g. ``) and multi-modal inputs (e.g. the raw input image) based on the outputs of HF processor. -Here are the main features of [BaseMultiModalProcessor][vllm.multimodal.processing.BaseMultiModalProcessor]: +In vLLM's rendering pipeline (see [BaseRenderer][vllm.renderers.base.BaseRenderer]), tokenization is performed as a separate step before multi-modal processing. Therefore, `BaseMultiModalProcessor` needs to recreate the output of calling HF processor end-to-end, while not being able to see the original text. This is achieved through **Dummy Input Text** and **Prompt Update Detection**. + +## Dummy Input Text + +Since Transformers 5.10, `ProcessorMixin` now allows multi-modal inputs to be passed by themselves. However, certain subclasses (such as `ChameleonProcessor`) and older out-of-tree implementations may still define their own `__call__` method that assumes the presence of text with corresponding placeholder tokens. This causes a problem as we don't have the original text anymore to pass to these processors. + +To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text] returns so that [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] passes it to the HF processor together with the multi-modal inputs to obtain the processed multi-modal data. + +Similarly, since the multi-modal data extracted by vLLM may not match what a specific HF processor expects, [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] allows each model to adapt the inputs via [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data] (e.g. renaming keys like `audios` to `audio` or injecting extra keyword arguments such as `sampling_rate`) and the outputs via [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data], without having to reimplement the entire method. ## Prompt Update Detection @@ -13,46 +21,9 @@ One of the main responsibilities of HF processor is to update the prompt with pl The information about which tokens have been updated is key to finding the correspondence between placeholder feature tokens and multi-modal inputs. -In vLLM, this information is specified using [PromptUpdate][vllm.multimodal.processing.PromptUpdate] in [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates]. We can automatically detect whether HF has updated the prompt by checking the existence of the updated tokens. +Since we call HF processor without the input text, we have to perform this update by ourselves. In vLLM, we represent the necessary information using [PromptUpdate][vllm.multimodal.processing.PromptUpdate] in [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates], and apply them via [_apply_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._apply_prompt_updates]. -## Tokenized Prompt Inputs - -To enable tokenization in a separate process, we support passing input token IDs alongside multi-modal data. - -### The problem - -Consider that HF processors follow these main steps: - -1. Tokenize the text -2. Process multi-modal inputs -3. Perform prompt updates - -And we require that: - -- For text + multi-modal inputs, apply all steps 1--3. -- For tokenized + multi-modal inputs, apply only steps 2--3. - -How can we achieve this without rewriting HF processors? We can try to call the HF processor several times on different inputs: - -- For text + multi-modal inputs, simply call the HF processor directly. -- For tokenized + multi-modal inputs, call the processor only on the multi-modal inputs. - -While HF processors support text + multi-modal inputs natively, this is not so for tokenized + multi-modal inputs: an error is thrown if the number of input placeholder tokens do not correspond to the number of multi-modal inputs. - -Moreover, since the tokenized text has not passed through the HF processor, we have to apply Step 3 by ourselves to keep the output tokens and multi-modal data consistent with each other. - -### Dummy text - -We work around the first issue by requiring each model to define how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text]. This lets us generate dummy text corresponding to the multi-modal inputs and input them together to obtain the processed multi-modal data. - -### Automatic prompt updating - -We address the second issue by implementing model-agnostic code in -[_apply_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._apply_prompt_updates] to automatically update the prompt with feature placeholder tokens based on the specification outputted by [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates]. - -### Summary - -With the help of dummy text and automatic prompt updating, our multi-modal processor can finally accept both text and token prompts with multi-modal data. The detailed logic is shown in [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]. +Some HF processors additionally transform the prompt itself regardless of the multi-modal inputs (such as `ChameleonProcessor` appending a sep token for chat mode). Since the prompt tokens likewise bypass the HF processor, such transformations are replicated via [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt] before the prompt updates are located or applied. ## Processor Output Caching @@ -60,8 +31,6 @@ Some HF processors, such as the one for Qwen2-VL, are [very slow](https://github When new data is passed in, we first check which items are in the cache, and which ones are missing. The missing items are passed into the HF processor in a single batch and cached, before being merged with the existing items in the cache. -Since we only process the missing multi-modal data items, the number of input placeholder tokens no longer corresponds to the number of the multi-modal inputs, so they can't be passed alongside the text prompt to HF processor. Therefore, we process the text and multi-modal inputs separately, using [dummy text](#dummy-text) to avoid HF errors. Since this skips HF's prompt updating code, we apply [automatic prompt updating](#automatic-prompt-updating) afterwards to keep the output tokens and multi-modal data consistent with each other. - ## Speeding Up Multi‑Modal Data Processing ### Fused Normalisation on the Device diff --git a/requirements/common.txt b/requirements/common.txt index 385dd28dac..a1f8dbea3c 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -7,7 +7,7 @@ requests >= 2.26.0 tqdm blake3 py-cpuinfo -transformers >= 5.5.3 +transformers >= 5.10.4 huggingface_hub >= 1.28.0 tokenizers >= 0.21.1 # Required for fast incremental detokenization. safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611 diff --git a/tests/models/multimodal/processing/test_audio_in_video.py b/tests/models/multimodal/processing/test_audio_in_video.py index 67235cfc88..df6eea0f38 100644 --- a/tests/models/multimodal/processing/test_audio_in_video.py +++ b/tests/models/multimodal/processing/test_audio_in_video.py @@ -136,7 +136,7 @@ def test_use_audio_in_video_without_audio_track(model_id: str) -> None: # no audio track" case. (The audio limit above is 1, not 0, so that # `mm_counts["audio"]` is still declared for this request; a 0 limit # would make vLLM omit the "audio" key from `mm_counts` entirely, which - # trips an unrelated assertion in `_apply_hf_processor_mm_only`.) + # trips an unrelated assertion in `_apply_hf_processor_main`.) mm_data = {"video": [video]} with pytest.raises(ValueError, match="doesn't have audio track"): diff --git a/tests/models/multimodal/processing/test_audioflamingo3.py b/tests/models/multimodal/processing/test_audioflamingo3.py index c7dffa90de..f5920a3be4 100644 --- a/tests/models/multimodal/processing/test_audioflamingo3.py +++ b/tests/models/multimodal/processing/test_audioflamingo3.py @@ -105,9 +105,9 @@ def test_audio_chunk_counting(mock_ctx): audio_2 = np.zeros(75 * sr) mm_data = {"audio": [audio_1, audio_2]} - prompt = "<|user|>Listen.<|end|>" - processed = processor._call_hf_processor(prompt, mm_data, {}) + mm_items = processor.info.parse_mm_data(mm_data, validate=False) + processed = processor._apply_hf_processor_main(mm_items, {}) chunk_counts = processed["chunk_counts"] diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index ce79ce2435..7755fc74a8 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -385,17 +385,19 @@ def test_processing_correctness( num_batches: int, simplify_rate: float, ): - if model_id == "google/gemma-3n-E2B-it": - pytest.skip("Fix later") - if model_id == "OpenGVLab/InternVL2-2B": - pytest.skip("Fix later") if model_id == "openvla/openvla-7b": pytest.skip( "OpenVLA uses a custom vLLM processor because its HF remote " "processor is incompatible with current Transformers." ) if model_id == "jinaai/jina-reranker-m0": - pytest.skip("Fix later") + pytest.skip( + "JinaVL reverses the multi-modal item order inside " + "_apply_hf_processor_main to match its document-before-query " + "score template, which breaks the positional correspondence " + "between multi-modal kwargs and hashes assumed by the " + "processing cache. Needs a fix that keeps both consistent." + ) if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602": pytest.skip( "Voxtral Realtime doesn't make use of any place-holder " @@ -403,22 +405,12 @@ def test_processing_correctness( "correctness test as is. Let's revisit adapting this " "test once more realtime models exist." ) - if model_id == "CohereLabs/cohere-transcribe-03-2026": - pytest.skip("Fix later") if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"): pytest.skip( "MOSS-Audio uses a custom processor that dynamically expands " "audio placeholders from processed audio lengths. Its vLLM " "processor paths are covered by test_moss_audio.py." ) - # TODO: Remove when transformers 5.15.0 is released, which contains - # https://github.com/huggingface/transformers/pull/47483. - if model_id == "microsoft/VibeVoice-ASR-HF": - pytest.skip( - "VibeVoice ASR requires audio as a positional argument and hence " - "cannot pass the processing correctness test as is. Its generation " - "is covered by test_transformers_audio.py." - ) if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct": pytest.skip( "LLaVA-OneVision-2 video processing routes frames through custom " diff --git a/tests/models/multimodal/processing/test_llava_onevision2.py b/tests/models/multimodal/processing/test_llava_onevision2.py index 817d9a4f67..5a84d049fd 100644 --- a/tests/models/multimodal/processing/test_llava_onevision2.py +++ b/tests/models/multimodal/processing/test_llava_onevision2.py @@ -202,7 +202,7 @@ def test_validate_relative_bare_path_blocked(): # --------------------------------------------------------------------------- # Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps) # -# Non-codec videos reach _call_hf_processor as a ``(frames_ndarray, metadata)`` +# Non-codec videos reach _apply_hf_processor_main as a ``(frames_ndarray, metadata)`` # tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real # ``video_url`` inputs, or by the dummy-inputs builder during profiling # (``video_needs_metadata=True``). The helper materialises PIL frames and diff --git a/tests/models/multimodal/processing/test_qwen3_vl.py b/tests/models/multimodal/processing/test_qwen3_vl.py index a1ab94ca43..2465a528ef 100644 --- a/tests/models/multimodal/processing/test_qwen3_vl.py +++ b/tests/models/multimodal/processing/test_qwen3_vl.py @@ -103,7 +103,7 @@ def test_processor_multi_video( """Verify that multi-video processing produces correct placeholders. This exercises the token-level replacement path in - ``_call_hf_processor`` which avoids the quadratic text-level + ``_apply_hf_processor_main`` which avoids the quadratic text-level prompt expansion. """ ctx = build_model_context( @@ -153,7 +153,7 @@ def test_processor_multi_video_list_kwargs( ``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not crash. - Before the fix, ``_call_hf_processor`` copied the whole kwargs to every + Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every video without slicing, so ``_get_video_second_idx`` received the list where a scalar was expected and raised ``TypeError``. """ diff --git a/vllm/model_executor/models/audioflamingo3.py b/vllm/model_executor/models/audioflamingo3.py index c5583b669b..43e6212c7a 100644 --- a/vllm/model_executor/models/audioflamingo3.py +++ b/vllm/model_executor/models/audioflamingo3.py @@ -377,35 +377,41 @@ class AudioFlamingo3MultiModalDataParser(MultiModalDataParser): class AudioFlamingo3MultiModalProcessor( BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo] ): - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _preprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, Any], - ) -> BatchFeature: - processor_mm_data = dict(mm_data) - audios = processor_mm_data.pop("audios", None) + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + mm_data = dict(mm_data) + audios = mm_data.pop("audios", None) if audios is not None: - processor_mm_data["audio"] = audios + mm_data["audio"] = audios - outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=processor_mm_data, - mm_kwargs=mm_kwargs, - ) + return mm_data, hf_processor_mm_kwargs - if "input_features_mask" in outputs: - outputs["feature_attention_mask"] = outputs.pop("input_features_mask") + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if "input_features_mask" in processed_data: + processed_data["feature_attention_mask"] = processed_data.pop( + "input_features_mask" + ) - audio_data = processor_mm_data.get("audio") + audio_data = mm_data.get("audio") if audio_data is None: - return outputs + return processed_data audio_list = audio_data if isinstance(audio_data, list) else [audio_data] if len(audio_list) == 0: - return outputs + return processed_data - processor = self.info.get_hf_processor(**mm_kwargs) + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) feature_extractor = processor.feature_extractor sampling_rate = feature_extractor.sampling_rate chunk_length = feature_extractor.chunk_length @@ -423,8 +429,9 @@ class AudioFlamingo3MultiModalProcessor( n_win = max_windows chunk_counts.append(n_win) - outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long) - return outputs + processed_data["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long) + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/blip2.py b/vllm/model_executor/models/blip2.py index 6b1be9860a..cc7fcd335d 100644 --- a/vllm/model_executor/models/blip2.py +++ b/vllm/model_executor/models/blip2.py @@ -467,24 +467,6 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]): class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - if not mm_data: - # HF processor always adds placeholders even when there's no image - tokenizer = self.info.get_tokenizer() - prompt_ids = tokenizer.encode(prompt) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") - - return super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/chameleon.py b/vllm/model_executor/models/chameleon.py index af147b7f1a..fa9cc13f0b 100644 --- a/vllm/model_executor/models/chameleon.py +++ b/vllm/model_executor/models/chameleon.py @@ -135,34 +135,16 @@ class ChameleonDummyInputsBuilder(BaseDummyInputsBuilder[ChameleonProcessingInfo class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - if not mm_data: - prompt_ids = self.info.get_tokenizer().encode(prompt) - prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + # ChameleonProcessor requires text corresponding to the images + return self.dummy_inputs.get_dummy_text(mm_counts) - return super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) - - def _apply_hf_processor_tokens_only( - self, - prompt_tokens: list[int], - ) -> list[int]: + def _postprocess_prompt(self, prompt: list[int]) -> list[int]: # HF processor adds sep token for chat mode tokenizer = self.info.get_tokenizer() - vocab = tokenizer.get_vocab() + sep_token_id = tokenizer.get_vocab()[tokenizer.sep_token] # type: ignore - sep_token_id = vocab[tokenizer.sep_token] # type: ignore - - return prompt_tokens + [sep_token_id] + return [*prompt, sep_token_id] def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/cheers.py b/vllm/model_executor/models/cheers.py index b9e88e1055..2b934f3867 100644 --- a/vllm/model_executor/models/cheers.py +++ b/vllm/model_executor/models/cheers.py @@ -16,7 +16,6 @@ import torch import torch.nn as nn import torch.nn.functional as F from einops import rearrange -from transformers import BatchFeature from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions @@ -492,14 +491,6 @@ class CheersDummyInputsBuilder(BaseDummyInputsBuilder[CheersProcessingInfo]): class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]): """Multimodal processor for Cheers model.""" - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) - def _get_prompt_updates( self, mm_items: MultiModalDataItems, diff --git a/vllm/model_executor/models/cohere2_vision.py b/vllm/model_executor/models/cohere2_vision.py index 147d462b23..6c13203b27 100644 --- a/vllm/model_executor/models/cohere2_vision.py +++ b/vllm/model_executor/models/cohere2_vision.py @@ -222,24 +222,23 @@ class Cohere2VisionDummyInputsBuilder( class Cohere2VisionMultiModalProcessor( BaseMultiModalProcessor[Cohere2VisionProcessingInfo] ): - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _postprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, ) -> BatchFeature: - processed_outputs = super()._call_hf_processor( - prompt, - mm_data, - mm_kwargs, - ) + if not mm_data: + return processed_data - # Ensure num_patches is available for proper tensor splitting if ( - "num_patches" not in processed_outputs + "num_patches" not in processed_data and (images := mm_data.get("images")) is not None ): - hf_processor = self.info.get_hf_processor(**mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) # Fallback calculation if HF processor didn't provide num_patches mm_items = self.info.parse_mm_data({"image": images}, validate=False) @@ -250,13 +249,13 @@ class Cohere2VisionMultiModalProcessor( image_width=parsed_images.get_image_size(i).width, image_height=parsed_images.get_image_size(i).height, processor=hf_processor, - mm_kwargs=mm_kwargs, + mm_kwargs=hf_processor_mm_kwargs, ) for i in range(len(parsed_images)) ] - processed_outputs["num_patches"] = torch.tensor(num_patches) + processed_data["num_patches"] = torch.tensor(num_patches) - return processed_outputs + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/cohere_asr.py b/vllm/model_executor/models/cohere_asr.py index 9cbf9bb205..b4e72d5b3f 100644 --- a/vllm/model_executor/models/cohere_asr.py +++ b/vllm/model_executor/models/cohere_asr.py @@ -8,7 +8,7 @@ from typing import Any, ClassVar import torch import torch.nn.functional as F from torch import nn -from transformers import PretrainedConfig +from transformers import BatchFeature, PretrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig @@ -45,9 +45,12 @@ from vllm.multimodal.processing import ( BaseDummyInputsBuilder, BaseProcessingInfo, EncDecMultiModalProcessor, + ProcessorInputs, PromptReplacement, PromptUpdate, + TimingContext, ) +from vllm.multimodal.processing.processor import MultiModalProcessingInfo from vllm.renderers import TokenizeParams from vllm.tokenizers import cached_tokenizer_from_config from vllm.transformers_utils.processors.cohere_asr import ( @@ -1941,27 +1944,51 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing ) -> list[int]: return [0] - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _preprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ): - if mm_data: - feature_extractor = self.info.get_feature_extractor(**mm_kwargs) - mm_data = dict(audio=dict(mm_data).pop("audios")) - mm_kwargs = dict( - **mm_kwargs, - sampling_rate=feature_extractor.sampling_rate, - ) - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + + mm_data = dict(mm_data) + mm_data["audio"] = mm_data.pop("audios") + + hf_processor_mm_kwargs = dict( + **hf_processor_mm_kwargs, + sampling_rate=feature_extractor.sampling_rate, ) - if "labels" in processed_outputs: - processed_outputs["input_ids"] = processed_outputs.pop("labels") - return processed_outputs + + return mm_data, hf_processor_mm_kwargs + + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if "labels" in processed_data: + processed_data["input_ids"] = processed_data.pop("labels") + + return processed_data + + def _cached_apply_hf_processor( + self, + inputs: ProcessorInputs, + timing_ctx: TimingContext, + ) -> MultiModalProcessingInfo: + # Dithering injects noise into the extracted features, so the + # feature extractor is not a pure function of its input. Since the + # processing cache assumes that processor outputs are invariant + # across calls, bypass the cache when dithering is active. + preproc = self.info.get_hf_config().preprocessor + if preproc.get("dither", 1e-05) > 0: + return self._apply_hf_processor(inputs, timing_ctx) + + return super()._cached_apply_hf_processor(inputs, timing_ctx) def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/colmodernvbert.py b/vllm/model_executor/models/colmodernvbert.py index ffa10452c5..89e55d40e2 100644 --- a/vllm/model_executor/models/colmodernvbert.py +++ b/vllm/model_executor/models/colmodernvbert.py @@ -154,16 +154,25 @@ class ColModernVBertDummyInputsBuilder( class ColModernVBertMultiModalProcessor( BaseMultiModalProcessor[ColModernVBertProcessingInfo], ): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + tokenizer = self.info.get_tokenizer() assert isinstance(tokenizer, HfTokenizer) text_encoding = tokenizer( - prompt, + prompt_text, return_tensors="pt", ) result = BatchFeature(data=dict(text_encoding)) @@ -183,7 +192,9 @@ class ColModernVBertMultiModalProcessor( ) result.update(image_outputs) - return result + processed_data = result + processed_data.update(passthrough_data) + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/colpali.py b/vllm/model_executor/models/colpali.py index aaf3b307a1..f31d63c006 100644 --- a/vllm/model_executor/models/colpali.py +++ b/vllm/model_executor/models/colpali.py @@ -28,6 +28,7 @@ from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.parse import MultiModalDataItems from .interfaces import SupportsLateInteraction from .interfaces_base import default_pooling_type @@ -59,24 +60,31 @@ class ColPaliProcessingInfo(PaliGemmaProcessingInfo): class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor): """Multimodal processor for ColPali.""" - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - if mm_data: - # The ColPali tokenizer_config.json ships with a small default - # max_length (50) that truncates the 1024 image tokens inserted - # by PaliGemmaProcessor, causing a token-count mismatch. - # vLLM enforces its own max_model_len, so we disable HF - # truncation to keep all image + text tokens intact. - return self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**mm_kwargs), - dict(text=prompt, **mm_data), - dict(**mm_kwargs, truncation=False), - ) - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + # The ColPali tokenizer_config.json ships with a small default + # max_length (50) that truncates the 1024 image tokens inserted + # by PaliGemmaProcessor, causing a token-count mismatch. + # vLLM enforces its own max_model_len, so we disable HF + # truncation to keep all image + text tokens intact. + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + mm_data, + dict(**hf_processor_mm_kwargs, truncation=False), + ) + processed_data.update(passthrough_data) + return processed_data @default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL") diff --git a/vllm/model_executor/models/deepseek_ocr.py b/vllm/model_executor/models/deepseek_ocr.py index 5365b031f9..8365c9069e 100644 --- a/vllm/model_executor/models/deepseek_ocr.py +++ b/vllm/model_executor/models/deepseek_ocr.py @@ -287,27 +287,34 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing class DeepseekOCRMultiModalProcessor( BaseMultiModalProcessor[DeepseekOCRProcessingInfo] ): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + if mm_data: - processed_outputs = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**mm_kwargs), - dict(prompt=prompt, **mm_data), - mm_kwargs, + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(prompt=prompt_text, **mm_data), + hf_processor_mm_kwargs, ) else: tokenizer = self.info.get_tokenizer() assert isinstance(tokenizer, HfTokenizer) - processed_outputs = tokenizer( - prompt, add_special_tokens=True, return_tensors="pt" + processed_data = tokenizer( + prompt_text, add_special_tokens=True, return_tensors="pt" ) - return processed_outputs + processed_data.update(passthrough_data) + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/deepseek_ocr2.py b/vllm/model_executor/models/deepseek_ocr2.py index d19af4713b..0e907a344a 100644 --- a/vllm/model_executor/models/deepseek_ocr2.py +++ b/vllm/model_executor/models/deepseek_ocr2.py @@ -158,27 +158,35 @@ class DeepseekOCR2DummyInputsBuilder( class DeepseekOCR2MultiModalProcessor( BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo] ): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + if mm_data: - processed_outputs = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**mm_kwargs), - dict(prompt=prompt, **mm_data), - mm_kwargs, + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(prompt=prompt_text, **mm_data), + hf_processor_mm_kwargs, ) else: tokenizer = self.info.get_tokenizer() assert isinstance(tokenizer, HfTokenizer) - processed_outputs = tokenizer( - prompt, add_special_tokens=True, return_tensors="pt" + processed_data = tokenizer( + prompt_text, add_special_tokens=True, return_tensors="pt" ) - return processed_outputs + processed_data.update(passthrough_data) + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/deepseek_vl2.py b/vllm/model_executor/models/deepseek_vl2.py index ac83b6028a..e77dd3d13f 100644 --- a/vllm/model_executor/models/deepseek_vl2.py +++ b/vllm/model_executor/models/deepseek_vl2.py @@ -43,7 +43,6 @@ from vllm.multimodal.processing.processor import ( ) from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config -from vllm.tokenizers.hf import HfTokenizer from vllm.transformers_utils.configs.deepseek_vl2 import ( DeepseekVLV2Config, MlpProjectorConfig, @@ -236,28 +235,23 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing class DeepseekVL2MultiModalProcessor( BaseMultiModalProcessor[DeepseekVL2ProcessingInfo] ): - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _postprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, ) -> BatchFeature: if not mm_data: - tokenizer = self.info.get_tokenizer() - assert isinstance(tokenizer, HfTokenizer) - return tokenizer(prompt, add_special_tokens=True, return_tensors="pt") + return processed_data - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, + processed_data["num_patches"] = ( + processed_data["images_spatial_crop"].prod(-1) + 1 ) - processed_outputs["num_patches"] = ( - processed_outputs["images_spatial_crop"].prod(-1) + 1 - ) - - return processed_outputs + return processed_data def _get_mm_fields_config( self, @@ -315,7 +309,7 @@ class DeepseekVL2MultiModalProcessor( self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> tuple[list[int], MultiModalProcessingInfo, bool]: + ) -> MultiModalProcessingInfo: # The processor logic is different for len(images) <= 2 vs > 2 # Since the processing cache assumes that the processor output is # invariant of how many images are passed per prompt, we only diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index 4b495fae7a..a1de2ae54e 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -1110,21 +1110,20 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin pixel_values = pixel_values.to(hf_config.dtype) return pixel_values - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - # when the prompt is not empty but the multimodal data is empty, - # directly invoke the tokenizer. - if "images" not in mm_data and "videos" not in mm_data and prompt != "": - tokenizer = self.info.get_tokenizer() - prompt_ids = tokenizer.encode(prompt) - tokenizer_output = BatchFeature( - dict(input_ids=[prompt_ids]), tensor_type="pt" - ) - return tokenizer_output + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if "images" not in mm_data and "videos" not in mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) if "images" not in mm_data: mm_data["images"] = [] @@ -1132,7 +1131,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin mm_data["videos"] = [] # Check if HF processor supports video metadata - hf_processor = self.info.get_hf_processor(**mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) supports_video_metadata = getattr( hf_processor, "supports_video_metadata", False ) @@ -1149,8 +1148,10 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin processor_output = self.info.ctx.call_hf_processor( hf_processor, - dict(text=[prompt], images=mm_data["images"], videos=mm_data["videos"]), - mm_kwargs, + dict( + text=[prompt_text], images=mm_data["images"], videos=mm_data["videos"] + ), + hf_processor_mm_kwargs, ) # Divide the processor_output into two modalities: image and video. @@ -1158,7 +1159,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin pixel_values = processor_output["images"] if pixel_values is not None: processor_output["images"] = self._pixel_values_norm( - pixel_values, mm_kwargs + pixel_values, hf_processor_mm_kwargs ) for key in list(processor_output.keys()): if processor_output[key] is None: @@ -1184,7 +1185,9 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin ] del processor_output["images"] - return processor_output + processed_data = processor_output + processed_data.update(passthrough_data) + return processed_data def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/fireredasr2.py b/vllm/model_executor/models/fireredasr2.py index 670e44cb86..f652eb7518 100644 --- a/vllm/model_executor/models/fireredasr2.py +++ b/vllm/model_executor/models/fireredasr2.py @@ -228,27 +228,36 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing class FireRedASR2MultiModalProcessor( BaseMultiModalProcessor[FireRedASR2ProcessingInfo] ): - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _preprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - if mm_data: - feature_extractor = self.info.get_feature_extractor(**mm_kwargs) - mm_data = dict(audio=mm_data.pop("audios")) - mm_kwargs = dict( - **mm_kwargs, - sampling_rate=feature_extractor.sampling_rate, - ) - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + + mm_data = dict(mm_data) + mm_data["audio"] = mm_data.pop("audios") + + hf_processor_mm_kwargs = dict( + **hf_processor_mm_kwargs, + sampling_rate=feature_extractor.sampling_rate, ) - if "labels" in processed_outputs: - processed_outputs["input_ids"] = processed_outputs.pop("labels") - return processed_outputs + + return mm_data, hf_processor_mm_kwargs + + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if "labels" in processed_data: + processed_data["input_ids"] = processed_data.pop("labels") + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/fireredlid.py b/vllm/model_executor/models/fireredlid.py index 32efe3dc0e..2225d70f9e 100644 --- a/vllm/model_executor/models/fireredlid.py +++ b/vllm/model_executor/models/fireredlid.py @@ -451,27 +451,36 @@ class FireRedLIDMultiModalProcessor( # Dummy encoder prompt for profiling (encoder only processes audio). return [0] - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _preprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - if mm_data: - feature_extractor = self.info.get_feature_extractor(**mm_kwargs) - mm_data = dict(audio=mm_data.pop("audios")) - mm_kwargs = dict( - **mm_kwargs, - sampling_rate=feature_extractor.sampling_rate, - ) - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + + mm_data = dict(mm_data) + mm_data["audio"] = mm_data.pop("audios") + + hf_processor_mm_kwargs = dict( + **hf_processor_mm_kwargs, + sampling_rate=feature_extractor.sampling_rate, ) - if "labels" in processed_outputs: - processed_outputs["input_ids"] = processed_outputs.pop("labels") - return processed_outputs + + return mm_data, hf_processor_mm_kwargs + + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if "labels" in processed_data: + processed_data["input_ids"] = processed_data.pop("labels") + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/funasr.py b/vllm/model_executor/models/funasr.py index 205cb5c0e6..e15cde2d75 100644 --- a/vllm/model_executor/models/funasr.py +++ b/vllm/model_executor/models/funasr.py @@ -44,9 +44,12 @@ from vllm.multimodal.processing import ( BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, + ProcessorInputs, PromptReplacement, PromptUpdate, + TimingContext, ) +from vllm.multimodal.processing.processor import MultiModalProcessingInfo from vllm.transformers_utils.processor import cached_processor_from_config from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -747,27 +750,50 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]): class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]): - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _preprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - if mm_data: - feature_extractor = self.info.get_feature_extractor(**mm_kwargs) - mm_data = dict(audio=mm_data.pop("audios")) - mm_kwargs = dict( - **mm_kwargs, - sampling_rate=feature_extractor.sampling_rate, - ) - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + + mm_data = dict(mm_data) + mm_data["audio"] = mm_data.pop("audios") + + hf_processor_mm_kwargs = dict( + **hf_processor_mm_kwargs, + sampling_rate=feature_extractor.sampling_rate, ) - if "labels" in processed_outputs: - processed_outputs["input_ids"] = processed_outputs.pop("labels") - return processed_outputs + + return mm_data, hf_processor_mm_kwargs + + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if "labels" in processed_data: + processed_data["input_ids"] = processed_data.pop("labels") + + return processed_data + + def _cached_apply_hf_processor( + self, + inputs: ProcessorInputs, + timing_ctx: TimingContext, + ) -> MultiModalProcessingInfo: + # Dithering injects noise into the extracted features, so the + # feature extractor is not a pure function of its input. Since the + # processing cache assumes that processor outputs are invariant + # across calls, bypass the cache when dithering is active. + if self.info.get_feature_extractor().dither > 0: + return self._apply_hf_processor(inputs, timing_ctx) + + return super()._cached_apply_hf_processor(inputs, timing_ctx) def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/funaudiochat.py b/vllm/model_executor/models/funaudiochat.py index 255e691df3..8a0cf1364c 100644 --- a/vllm/model_executor/models/funaudiochat.py +++ b/vllm/model_executor/models/funaudiochat.py @@ -612,20 +612,31 @@ class FunAudioChatDummyInputsBuilder( class FunAudioChatMultiModalProcessor( BaseMultiModalProcessor[FunAudioChatProcessingInfo] ): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + tokenizer = self.info.get_tokenizer() - input_ids = torch.tensor([tokenizer.encode(prompt)]) + input_ids = torch.tensor([tokenizer.encode(prompt_text)]) audios = mm_data.get("audios", []) if not audios: - return BatchFeature({"input_ids": input_ids}) + processed_data = BatchFeature({"input_ids": input_ids}) + processed_data.update(passthrough_data) + return processed_data - feature_extractor = self.info.get_feature_extractor(**mm_kwargs) + feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) sr = int(feature_extractor.sampling_rate) min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400) @@ -676,7 +687,9 @@ class FunAudioChatMultiModalProcessor( "feature_exist_mask": torch.ones((len(wavs),), dtype=torch.bool), } - return BatchFeature({"input_ids": input_ids, **mm_inputs}) + processed_data = BatchFeature({"input_ids": input_ids, **mm_inputs}) + processed_data.update(passthrough_data) + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index 2176b023fa..d698483cb8 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -261,39 +261,38 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]): class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]): - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _postprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, ) -> BatchFeature: - processed_outputs = super()._call_hf_processor( - prompt, - mm_data, - mm_kwargs, - ) + if not mm_data: + return processed_data - # HF processor pops the `num_crops` kwarg, which is needed by vLLM if (images := mm_data.get("images")) is not None: mm_items = self.info.parse_mm_data({"image": images}, validate=False) parsed_images = mm_items.get_items("image", ImageProcessorItems) image_sizes = [ parsed_images.get_image_size(i) for i in range(len(parsed_images)) ] - hf_processor = self.info.get_hf_processor(**mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) num_crops = [ self.info.get_num_crops( image_width=size.width, image_height=size.height, processor=hf_processor, - mm_kwargs=mm_kwargs, + mm_kwargs=hf_processor_mm_kwargs, ) for size in image_sizes ] - processed_outputs["num_patches"] = torch.tensor(num_crops) + 1 + processed_data["num_patches"] = torch.tensor(num_crops) + 1 - return processed_outputs + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/gemma3n_mm.py b/vllm/model_executor/models/gemma3n_mm.py index 787642fa1e..a473d9f437 100644 --- a/vllm/model_executor/models/gemma3n_mm.py +++ b/vllm/model_executor/models/gemma3n_mm.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import inspect from collections.abc import Iterable, Mapping, Sequence from typing import Annotated, Any, Literal @@ -257,40 +258,69 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]): class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]): - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _preprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - # HF Transformers audio processor no longer accepts `audios` key. - # We pop `audios` and replace it with `audio` key to suppress - # the warning. + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + mm_data = dict(mm_data) if "audios" in mm_data: mm_data["audio"] = mm_data.pop("audios") - processed_outputs = super()._call_hf_processor( - prompt, - mm_data, - mm_kwargs, - ) - if "input_features" in processed_outputs: - # Padding enables audio_tower to run in batched mode - processed_outputs["input_features_padded"] = processed_outputs[ - "input_features" + return mm_data, hf_processor_mm_kwargs + + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if "input_features" in processed_data: + # The feature extractor pads every clip to the longest one in + # the batch, so each clip's frame count depends on the other + # clips it was processed with. Trim each clip to the number of + # frames it would have if processed alone so that each item's + # output (and thus its cache entry) is independent of the other + # items in the batch. `batch_audio_features` re-pads the + # per-item features so audio_tower can still run batched. + num_frames = [ + self._get_num_audio_frames_alone(len(audio)) + for audio in mm_data["audio"] + ] + processed_data["input_features_padded"] = [ + features[:n] + for features, n in zip(processed_data.pop("input_features"), num_frames) + ] + processed_data["input_features_mask"] = [ + mask[:n] + for mask, n in zip(processed_data["input_features_mask"], num_frames) ] - # Unpad features here since we need the output of each item to be - # independent of other items for the cache to work correctly - unpadded_features = [ - f[mask] - for f, mask in zip( - processed_outputs["input_features"], - processed_outputs["input_features_mask"], - ) - ] - processed_outputs["input_features"] = unpadded_features - return processed_outputs + return processed_data + + def _get_num_audio_frames_alone(self, num_samples: int) -> int: + """ + Get the number of mel frames that the feature extractor produces + for a single audio clip of the given length, independent of batch + padding. + """ + feature_extractor = self.info.get_feature_extractor() + call_params = inspect.signature(type(feature_extractor).__call__).parameters + max_length = call_params["max_length"].default + pad_to_multiple_of = call_params["pad_to_multiple_of"].default + + num_samples = min(num_samples, max_length) + remainder = num_samples % pad_to_multiple_of + if remainder: + num_samples += pad_to_multiple_of - remainder + + frame_size = feature_extractor.frame_length + 1 + num_frames = (num_samples - frame_size) // feature_extractor.hop_length + 1 + + return max(num_frames, 0) def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index 15774a9ad2..29b9a08aef 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -560,13 +560,22 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs) + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + + merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs) val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs) if val is not None and val not in _SUPPORTED_SOFT_TOKENS: @@ -611,15 +620,15 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): timestamps = [idx / fps for idx in frame_indices] # Process frames as images with max_soft_tokens=70 - video_mm_kwargs = dict(mm_kwargs) + video_mm_kwargs = dict(hf_processor_mm_kwargs) video_mm_kwargs["max_soft_tokens"] = _VIDEO_MAX_SOFT_TOKENS dummy_prompt = ("\t" + processor.image_token) * len(frames) - frame_outputs = super()._call_hf_processor( - prompt=dummy_prompt, - mm_data={"images": frames}, - mm_kwargs=video_mm_kwargs, + frame_outputs = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**video_mm_kwargs), + dict(text=dummy_prompt, **{"images": frames}), + video_mm_kwargs, ) # Remap HF key name @@ -660,7 +669,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): # split-replace bug where replacement text (which itself # contains <|video|> tokens) collides with later splits. vt = processor.video_token - parts = prompt.split(vt, len(video_replacements)) + parts = prompt_text.split(vt, len(video_replacements)) # NOTE: len(parts) <= len(video_replacements) + 1 parts_with_repl: list[str] = [] @@ -668,7 +677,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): parts_with_repl.extend([part, repl]) parts_with_repl.extend(parts[len(video_replacements) :]) - prompt = "".join(parts_with_repl) + prompt_text = "".join(parts_with_repl) video_outputs = { "pixel_values_videos": torch.cat(all_video_pixel_values, dim=0), @@ -712,45 +721,46 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): # NOTE: This requires a corresponding type annotation on the # HF side (Gemma4ProcessorKwargs.images_kwargs) so that # _merge_kwargs routes max_soft_tokens into images_kwargs. - patched_mm_kwargs = dict(mm_kwargs) + patched_mm_kwargs = dict(hf_processor_mm_kwargs) if val is not None and is_top_level_max_soft_tokens: patched_mm_kwargs["max_soft_tokens"] = val - processed_outputs = super()._call_hf_processor( - prompt, - mm_data, + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**patched_mm_kwargs), + dict(text=prompt_text, **mm_data), patched_mm_kwargs, ) # HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'. # Remap here to keep a single translation point. - if "image_position_ids" in processed_outputs: - processed_outputs["pixel_position_ids"] = processed_outputs.pop( + if "image_position_ids" in processed_data: + processed_data["pixel_position_ids"] = processed_data.pop( "image_position_ids" ) - if "input_features" in processed_outputs: + if "input_features" in processed_data: # Unpad per-item so each item's cache entry is # self-contained. The batched() field config in # _get_mm_fields_config stacks equal-length items and leaves # the rest as a list; _process_audio_input re-pads that list # to the batch's max length. - masks = processed_outputs["input_features_mask"] + masks = processed_data["input_features_mask"] unpadded_features = [ f[mask] for f, mask in zip( - processed_outputs["input_features"], + processed_data["input_features"], masks, ) ] unpadded_masks = [mask[mask] for mask in masks] - processed_outputs["input_features"] = unpadded_features - processed_outputs["input_features_padded"] = unpadded_features - processed_outputs["input_features_mask"] = unpadded_masks + processed_data["input_features"] = unpadded_features + processed_data["input_features_padded"] = unpadded_features + processed_data["input_features_mask"] = unpadded_masks - # Merge video outputs into the final result - combined_outputs = dict(processed_outputs, **video_outputs) - return BatchFeature(combined_outputs) + processed_data.update(video_outputs) + processed_data.update(passthrough_data) + + return processed_data def _get_mm_fields_config( self, @@ -815,7 +825,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): image_size = images.get_image_size(item_idx) # Resolve the effective max_soft_tokens by merging # per-prompt kwargs with the config-level defaults, - # consistent with how _call_hf_processor resolves it. + # consistent with how _apply_hf_processor_main resolves it. # Without this merge, a missing per-prompt override # would fall back to vision_cfg.default_output_length # instead of the config's mm_processor_kwargs default. diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 9bcca2f8bf..a4690e9a47 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -1576,32 +1576,39 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): return prepared_data, prepared_kwargs - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - mm_data = dict(mm_data) - if not mm_data: - tokenizer = self.info.get_tokenizer() - prompt_ids = tokenizer.encode(prompt, add_special_tokens=False) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - processor = self.info.get_hf_processor(**mm_kwargs) + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + + mm_data = dict(mm_data) + + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) use_direct_path = ( not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA ) if use_direct_path: prepared_data, prepared_kwargs = self._get_direct_path_inputs( - mm_data, mm_kwargs + mm_data, hf_processor_mm_kwargs ) - return super()._call_hf_processor( - prompt=prompt, - mm_data=prepared_data, - mm_kwargs=prepared_kwargs, + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**prepared_kwargs), + dict(text=prompt_text, **prepared_data), + prepared_kwargs, ) + processed_data.update(passthrough_data) + return processed_data if ( "videos" in mm_data @@ -1616,7 +1623,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): video_array, metadata = item # don't update mm_kwargs inplace - video_mm_kwargs = dict(**mm_kwargs) + video_mm_kwargs = dict(**hf_processor_mm_kwargs) video_mm_kwargs["do_sample_frames"] = metadata.get( "do_sample_frames", True ) @@ -1625,10 +1632,13 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): video_mm_data["videos"] = [[video_array]] video_mm_data["video_metadata"] = [[_to_video_metadata(metadata)]] - video_outputs = super()._call_hf_processor( - prompt="<|begin_of_video|><|video|><|end_of_video|>", - mm_data=video_mm_data, - mm_kwargs=video_mm_kwargs, + video_outputs = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**video_mm_kwargs), + dict( + text="<|begin_of_video|><|video|><|end_of_video|>", + **video_mm_data, + ), + video_mm_kwargs, ) input_ids = video_outputs.pop("input_ids") if swap_video_frame_tokens: @@ -1636,7 +1646,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): processor.video_token_id ) video_placeholder = processor.tokenizer.batch_decode(input_ids)[0] - prompt = prompt.replace( + prompt_text = prompt_text.replace( "<|begin_of_video|><|video|><|end_of_video|>", video_placeholder, 1, @@ -1652,20 +1662,20 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): video_outputs = dict() swap_video_frame_tokens = False - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=prompt_text, **mm_data), + hf_processor_mm_kwargs, ) if swap_video_frame_tokens: - input_ids = processed_outputs["input_ids"] + input_ids = processed_data["input_ids"] input_ids[input_ids == processor.video_token_id] = processor.image_token_id - processed_outputs["input_ids"] = input_ids - combined_outputs = dict( - processed_outputs, - **video_outputs, - ) - return BatchFeature(combined_outputs) + processed_data["input_ids"] = input_ids + + processed_data.update(video_outputs) + processed_data.update(passthrough_data) + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/glmasr.py b/vllm/model_executor/models/glmasr.py index d90bbb5634..fcf17c550c 100644 --- a/vllm/model_executor/models/glmasr.py +++ b/vllm/model_executor/models/glmasr.py @@ -750,46 +750,40 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"]) chunk_counts.append(min(n_chunks, max_windows)) return chunk_counts - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _preprocess_hf_mm_data( self, - prompt: str, - mm_data: dict[str, object], - mm_kwargs: Mapping[str, Any], - ) -> BatchFeature: - # Normalize input: handle deprecated key and list conversion. + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + mm_data = dict(mm_data) if "audios" in mm_data: mm_data["audio"] = mm_data.pop("audios") - audio = mm_data.get("audio", []) - audio_list = [audio] if audio and not isinstance(audio, list) else audio - - # Early return for text-only. - if not audio_list: - prompt_ids = self.info.get_tokenizer().encode(prompt) - prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") - - # Handle sampling_rate - feature_extractor = self.info.get_feature_extractor(**mm_kwargs) - mm_kwargs = dict( - **mm_kwargs, + feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + hf_processor_mm_kwargs = dict( + **hf_processor_mm_kwargs, sampling_rate=feature_extractor.sampling_rate, ) - # Call parent method - outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) + return mm_data, hf_processor_mm_kwargs - # Postprocess: rename mask and add chunk counts + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: # Handle different key names from different transformers versions - if "input_features_mask" in outputs: - outputs["feature_attention_mask"] = outputs.pop("input_features_mask") - elif "input_features_mask" not in outputs and "input_features" in outputs: + if "input_features_mask" in processed_data: + processed_data["feature_attention_mask"] = processed_data.pop( + "input_features_mask" + ) + elif "input_features" in processed_data: # If no mask is provided, create one from input_features - input_features = outputs["input_features"] + input_features = processed_data["input_features"] if isinstance(input_features, torch.Tensor): # Create a mask of all ones matching the sequence length mask = torch.ones( @@ -797,18 +791,22 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"]) input_features.shape[-1], dtype=torch.long, ) - outputs["feature_attention_mask"] = mask + processed_data["feature_attention_mask"] = mask - # Get processor for chunk counts calculation - processor = self.info.get_hf_processor(**mm_kwargs) + audio = mm_data.get("audio", []) + audio_list = [audio] if audio and not isinstance(audio, list) else audio + if audio_list: + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - # Override chunk counts calculation with GLM-ASR specific logic - chunk_counts = self._calculate_chunk_counts( - audio_list, processor.feature_extractor, processor - ) - outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long) + # Override chunk counts calculation with GLM-ASR specific logic + chunk_counts = self._calculate_chunk_counts( + audio_list, processor.feature_extractor, processor + ) + processed_data["chunk_counts"] = torch.tensor( + chunk_counts, dtype=torch.long + ) - return outputs + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/granite_speech.py b/vllm/model_executor/models/granite_speech.py index 626750bd96..9233e772d7 100644 --- a/vllm/model_executor/models/granite_speech.py +++ b/vllm/model_executor/models/granite_speech.py @@ -180,12 +180,14 @@ class GraniteSpeechMultiModalProcessor( ) ] - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _preprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: mm_data = dict(mm_data) audios = mm_data.pop("audios", []) @@ -193,21 +195,23 @@ class GraniteSpeechMultiModalProcessor( # GraniteSpeechFeatureExtractor accepts "audio" mm_data["audio"] = audios - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) + return mm_data, hf_processor_mm_kwargs + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: if "audio" in mm_data: # Calculate the number of audio tokens per entry in the batch; # This is used to split the batch back out after padding. audio_token_index = self.info.get_hf_config().audio_token_index - processed_outputs["audio_embed_sizes"] = ( - processed_outputs["input_ids"] == audio_token_index + processed_data["audio_embed_sizes"] = ( + processed_data["input_ids"] == audio_token_index ).sum(-1) - return processed_outputs + return processed_data class GraniteSpeechDummyInputsBuilder( diff --git a/vllm/model_executor/models/h2ovl.py b/vllm/model_executor/models/h2ovl.py index 40240d3e4e..dc63276d82 100644 --- a/vllm/model_executor/models/h2ovl.py +++ b/vllm/model_executor/models/h2ovl.py @@ -135,7 +135,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> tuple[list[int], MultiModalProcessingInfo, bool]: + ) -> MultiModalProcessingInfo: # The processor logic is different for len(images) <= 1 vs > 1 # Since the processing cache assumes that the processor output is # invariant of how many images are passed per prompt, we only diff --git a/vllm/model_executor/models/hunyuan_vision.py b/vllm/model_executor/models/hunyuan_vision.py index 4454ce47c1..ccc3f8fee9 100644 --- a/vllm/model_executor/models/hunyuan_vision.py +++ b/vllm/model_executor/models/hunyuan_vision.py @@ -731,28 +731,6 @@ class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - hf_processor = self.info.get_hf_processor(**mm_kwargs) - # HunYuanVLProcessor requires image placeholders wrapped with start/end tokens. - if mm_data.get("images") is not None and prompt: - img_tok = hf_processor.image_token - wrapped = ( - f"{hf_processor.image_start_token}{img_tok}" - f"{hf_processor.image_end_token}" - ) - if img_tok in prompt and wrapped not in prompt: - prompt = prompt.replace(img_tok, wrapped) - return self.info.ctx.call_hf_processor( - hf_processor, - dict(text=prompt, **mm_data), - mm_kwargs, - ) - def _get_prompt_updates( self, mm_items: MultiModalDataItems, diff --git a/vllm/model_executor/models/hyperclovax_vision.py b/vllm/model_executor/models/hyperclovax_vision.py index e469c02022..41387ad7e2 100644 --- a/vllm/model_executor/models/hyperclovax_vision.py +++ b/vllm/model_executor/models/hyperclovax_vision.py @@ -190,77 +190,71 @@ class HCXVisionDummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionProcessingInfo class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingInfo]): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + for video_idx, video_arr in enumerate(mm_data.get("videos", [])): if video_arr.dtype != np.uint8: mm_data["videos"][video_idx] = video_arr.astype(np.uint8) - processed_outputs = self.info.ctx.call_hf_processor( - hf_processor=self.info.get_hf_processor(**mm_kwargs), + images = mm_data.get("images") + videos = mm_data.get("videos") + + # batchify input as a single item + processed_data = self.info.ctx.call_hf_processor( + hf_processor=self.info.get_hf_processor(**hf_processor_mm_kwargs), data=dict( - text=prompt, - images=None, - videos=None, + images=None if images is None else [images], + videos=None if videos is None else [videos], ), - ) # text-only + ) - if len(mm_data) > 0: - images = mm_data.get("images") - videos = mm_data.get("videos") + for k, v in processed_data.items(): + if isinstance(v, list) and len(v) > 0: + assert len(v) == 1 + processed_data[k] = v[0] - # batchify input as a single item - _processed_outputs = self.info.ctx.call_hf_processor( - hf_processor=self.info.get_hf_processor(**mm_kwargs), - data=dict( - text=None, - images=None if images is None else [images], - videos=None if videos is None else [videos], - ), - ) # mm-only + if images: + processed_data["image_sizes_images"] = torch.tensor( + processed_data["image_sizes_images"] + ) + processed_data["vision_query_lengths_images"] = torch.tensor( + processed_data["vision_query_lengths_images"] + ) - for k, v in _processed_outputs.items(): - if isinstance(v, list) and len(v) > 0: - assert len(v) == 1 - _processed_outputs[k] = v[0] - - if images: - _processed_outputs["image_sizes_images"] = torch.tensor( - _processed_outputs["image_sizes_images"] - ) - _processed_outputs["vision_query_lengths_images"] = torch.tensor( - _processed_outputs["vision_query_lengths_images"] - ) - - if videos: - _idx_per_video = [ - 0, - *accumulate( - get_num_combined_frames(len(video)) for video in videos - ), + if videos: + _idx_per_video = [ + 0, + *accumulate(get_num_combined_frames(len(video)) for video in videos), + ] + processed_data["pixel_values_videos"] = [ + processed_data["pixel_values_videos"][ + _idx_per_video[i] : _idx_per_video[i + 1] ] - _processed_outputs["pixel_values_videos"] = [ - _processed_outputs["pixel_values_videos"][ + for i in range(len(videos)) + ] + processed_data["vision_query_lengths_videos"] = [ + torch.tensor( + processed_data["vision_query_lengths_videos"][ _idx_per_video[i] : _idx_per_video[i + 1] ] - for i in range(len(videos)) - ] - _processed_outputs["vision_query_lengths_videos"] = [ - torch.tensor( - _processed_outputs["vision_query_lengths_videos"][ - _idx_per_video[i] : _idx_per_video[i + 1] - ] - ) - for i in range(len(videos)) - ] + ) + for i in range(len(videos)) + ] - processed_outputs.update(_processed_outputs) + processed_data.update(passthrough_data) - return processed_outputs + return processed_data def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/hyperclovax_vision_v2.py b/vllm/model_executor/models/hyperclovax_vision_v2.py index e50faf8855..b1d64db9d9 100644 --- a/vllm/model_executor/models/hyperclovax_vision_v2.py +++ b/vllm/model_executor/models/hyperclovax_vision_v2.py @@ -261,34 +261,44 @@ class HCXVisionV2MultiModalProcessor( ): """Multimodal processor for HyperCLOVAX V2 (32B Think model).""" - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + images = mm_data.get("images") videos = mm_data.get("videos") # Get the HF processor - hf_processor = self.info.get_hf_processor(**mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) # Build data dict for HF processor (images/videos only) # NOTE: We pass the prompt as-is without token normalization. # Token expansion is handled by vLLM via _get_prompt_updates. data: dict[str, object] = dict( - text=prompt, + text=prompt_text, images=images, videos=videos, ) - processed_outputs = self.info.ctx.call_hf_processor( + processed_data = self.info.ctx.call_hf_processor( hf_processor=hf_processor, data=data, - kwargs=mm_kwargs, + kwargs=hf_processor_mm_kwargs, ) + processed_data.update(passthrough_data) - return processed_outputs + return processed_data def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index 226bfbff79..fbf0e7a7c3 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -239,51 +239,58 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]) class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - # Text-only input not supported in composite processor - if not (images := mm_data.get("images", [])): - prompt_ids = self.info.get_tokenizer().encode(prompt) - prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) image_processor = self.info.get_hf_processor().image_processor if getattr(image_processor, "backend", "pil") == "pil": - mm_kwargs = {"input_data_format": "channels_last", **mm_kwargs} + hf_processor_mm_kwargs = { + "input_data_format": "channels_last", + **hf_processor_mm_kwargs, + } - processed_outputs = super()._call_hf_processor( - prompt, + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), mm_data, - mm_kwargs, + hf_processor_mm_kwargs, ) + images = mm_data.get("images", []) mm_items = self.info.parse_mm_data({"image": images}, validate=False) parsed_images = mm_items.get_items("image", ImageProcessorItems) image_sizes = [ parsed_images.get_image_size(i) for i in range(len(parsed_images)) ] - hf_processor = self.info.get_hf_processor(**mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) num_patches = [ self.info.get_num_patches( image_width=size.width, image_height=size.height, processor=hf_processor, - mm_kwargs=mm_kwargs, + mm_kwargs=hf_processor_mm_kwargs, ) for size in image_sizes ] - processed_outputs["num_patches"] = torch.tensor(num_patches) + processed_data["num_patches"] = torch.tensor(num_patches) # Remove the extra batch dimension - processed_outputs["pixel_values"].squeeze_(0) - processed_outputs["pixel_attention_mask"].squeeze_(0) + processed_data["pixel_values"].squeeze_(0) + processed_data["pixel_attention_mask"].squeeze_(0) - return processed_outputs + processed_data.update(passthrough_data) + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/interns1.py b/vllm/model_executor/models/interns1.py index bb3ba6a556..2248911b20 100644 --- a/vllm/model_executor/models/interns1.py +++ b/vllm/model_executor/models/interns1.py @@ -332,19 +332,28 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo]) class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]): """Basic image-only MultiModalProcessor for InternS1-style models.""" - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + mm_data = dict(mm_data) videos = mm_data.pop("videos", []) images = mm_data.pop("images", []) assert isinstance(videos, list) assert isinstance(images, list) - hf_processor = self.info.get_hf_processor(**mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) tokenizer = hf_processor.tokenizer video_token_id = tokenizer.encode( hf_processor.video_token, add_special_tokens=False @@ -352,23 +361,29 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo assert len(video_token_id) == 1 video_token_id = video_token_id[0] - prompt = re.sub(hf_processor.image_token, "", prompt) - prompt = re.sub(hf_processor.video_token, "", prompt) + prompt_text = re.sub( + hf_processor.image_token, "", prompt_text + ) + prompt_text = re.sub( + hf_processor.video_token, "", prompt_text + ) image_outputs = {} if images: image_pixel_values = [] for image in images: - processed_outputs = super()._call_hf_processor( - prompt=hf_processor.image_token, - mm_data={"images": image}, - mm_kwargs=mm_kwargs, + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=hf_processor.image_token, **{"images": image}), + hf_processor_mm_kwargs, ) - image_pixel_values.append(processed_outputs.pop("pixel_values")) + image_pixel_values.append(processed_data.pop("pixel_values")) - input_ids = processed_outputs.pop("input_ids") + input_ids = processed_data.pop("input_ids") image_placeholder = tokenizer.batch_decode(input_ids)[0] - prompt = prompt.replace("", image_placeholder, 1) + prompt_text = prompt_text.replace( + "", image_placeholder, 1 + ) num_patches = [len(item) for item in image_pixel_values] image_outputs = { @@ -381,18 +396,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo if videos: video_pixel_values = [] for video in videos: - processed_outputs = super()._call_hf_processor( - prompt=hf_processor.video_token, - mm_data={"videos": video}, - mm_kwargs=mm_kwargs, + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=hf_processor.video_token, **{"videos": video}), + hf_processor_mm_kwargs, ) - video_pixel_values.append(processed_outputs.pop("pixel_values")) + video_pixel_values.append(processed_data.pop("pixel_values")) - input_ids = processed_outputs.pop("input_ids") + input_ids = processed_data.pop("input_ids") input_ids[input_ids == hf_processor.image_token_id] = video_token_id video_placeholder = tokenizer.batch_decode(input_ids)[0] - prompt = prompt.replace("", video_placeholder, 1) + prompt_text = prompt_text.replace( + "", video_placeholder, 1 + ) num_frames = [len(item) for item in video_pixel_values] video_outputs = { @@ -401,11 +418,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo "video_token_id": torch.tensor(video_token_id), } - prompt = re.sub("", hf_processor.image_token, prompt) - prompt = re.sub("", hf_processor.video_token, prompt) - text_outputs = tokenizer(prompt, return_tensors="pt") + prompt_text = re.sub( + "", hf_processor.image_token, prompt_text + ) + prompt_text = re.sub( + "", hf_processor.video_token, prompt_text + ) + text_outputs = tokenizer(prompt_text, return_tensors="pt") - return BatchFeature({**text_outputs, **image_outputs, **video_outputs}) + processed_data.update(text_outputs) + processed_data.update(image_outputs) + processed_data.update(video_outputs) + processed_data.update(passthrough_data) + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index 3895533439..ae7b05a553 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -220,27 +220,21 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]): class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]): """Basic image-only MultiModalProcessor for InternVL-style models.""" - def _call_hf_processor( + def _postprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, ) -> BatchFeature: - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) - - hf_processor = self.info.get_hf_processor(**mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) image_token_id = hf_processor.ctx_image_token_id # Since there may be extra tokens in the feature placeholders, # we need to pass the image token ID to the model to select the # tokens to merge from the vision encoder outputs - processed_outputs["image_token_id"] = torch.tensor(image_token_id) + processed_data["image_token_id"] = torch.tensor(image_token_id) - return processed_outputs + return processed_data def _get_image_fields_config(self, hf_inputs: BatchFeature): image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0)) @@ -455,19 +449,23 @@ class InternVLMultiModalProcessor( ): """InternVL MultiModalProcessor extended for video support""" - def _call_hf_processor( + def _postprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, ) -> BatchFeature: - processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs) + processed_data = super()._postprocess_hf_mm_data( + mm_data, + hf_processor_mm_kwargs, + processed_data, + ) - hf_processor = self.info.get_hf_processor(**mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) if (video_token_id := hf_processor.ctx_video_token_id) is not None: - processed_outputs["video_token_id"] = torch.tensor(video_token_id) + processed_data["video_token_id"] = torch.tensor(video_token_id) - return processed_outputs + return processed_data def _get_video_fields_config(self, hf_inputs: BatchFeature): video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0)) diff --git a/vllm/model_executor/models/jina_vl.py b/vllm/model_executor/models/jina_vl.py index c1cdf207e3..fdfc53def6 100644 --- a/vllm/model_executor/models/jina_vl.py +++ b/vllm/model_executor/models/jina_vl.py @@ -12,6 +12,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear from vllm.model_executor.layers.pooler import DispatchPooler from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.parse import MultiModalDataItems from vllm.sequence import IntermediateTensors from .interfaces import SupportsCrossEncoding, SupportsMultiModal, SupportsScoreTemplate @@ -54,19 +55,28 @@ class JinaVLScorer(nn.Module): class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - # NOTE: We should reverse the order of the mm_data because the - # query prompt is placed after the document prompt in the score - # template for JinaVLForRanking model, but in mm_data they are - # stored in the opposite order (query first, then document). + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + for _, value in mm_data.items(): value.reverse() - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + mm_data, + hf_processor_mm_kwargs, + ) + processed_data.update(passthrough_data) + return processed_data @MULTIMODAL_REGISTRY.register_processor( diff --git a/vllm/model_executor/models/kanana_v.py b/vllm/model_executor/models/kanana_v.py index 8899914997..8e55054d38 100644 --- a/vllm/model_executor/models/kanana_v.py +++ b/vllm/model_executor/models/kanana_v.py @@ -461,17 +461,21 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo]) def media_token_id(self) -> int: return self.info.get_hf_config().text_config.eos_token_id + 1 - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: """Run the underlying HF processor on text and image data.""" - # Text-only input is handled as a special case here. + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + if not mm_data or not mm_data.get("images", []): - prompt_ids = self.info.get_tokenizer().encode(prompt) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) # Images image_inputs = mm_data.get("images", []) @@ -493,7 +497,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo]) tokenizer = self.info.get_tokenizer() media_token = tokenizer.convert_ids_to_tokens([self.media_token_id])[0] - prompt_replaced = prompt.replace("", media_token) + prompt_replaced = prompt_text.replace("", media_token) input_ids = tokenizer.encode(prompt_replaced) input_ids = torch.tensor(input_ids) @@ -527,7 +531,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo]) image_token_thw=torch.tensor(image_meta["image_token_thw"]), pixel_sizes=torch.tensor(pixel_sizes), ) - return BatchFeature(combined_outputs, tensor_type="pt") + processed_data = BatchFeature(combined_outputs, tensor_type="pt") + processed_data.update(passthrough_data) + return processed_data def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index 7a504e83b7..bd7eb79f6f 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -1162,15 +1162,8 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]): class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - # Override to use the text path instead of token path to use the - # video-specific logic in processing_keye.py - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/keye_vl1_5.py b/vllm/model_executor/models/keye_vl1_5.py index 80a94373cb..8625fd9c01 100644 --- a/vllm/model_executor/models/keye_vl1_5.py +++ b/vllm/model_executor/models/keye_vl1_5.py @@ -378,15 +378,8 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo): class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - # Override to use the text path instead of token path to use the - # video-specific logic in processing_keye.py - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/kimi_audio.py b/vllm/model_executor/models/kimi_audio.py index 60534658ae..71cf389f98 100644 --- a/vllm/model_executor/models/kimi_audio.py +++ b/vllm/model_executor/models/kimi_audio.py @@ -37,6 +37,7 @@ from vllm.multimodal.parse import ( DictEmbeddingItems, ModalityData, ModalityDataItems, + MultiModalDataItems, MultiModalDataParser, ) from vllm.multimodal.processing import ( @@ -226,13 +227,20 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser): class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]): """vLLM multi-modal processor wrapper for Kimi-Audio.""" - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: """Call the HuggingFace processor.""" + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + # Convert mm_data format: {'audios': [...]} -> {'audio': ...} mm_data = dict(mm_data) audios = mm_data.pop("audios", []) @@ -252,11 +260,13 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn mm_data["audio"] = audio_arrays # Use the context's call_hf_processor for proper handling - return self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**mm_kwargs), - dict(text=prompt, **mm_data), - mm_kwargs, + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + mm_data, + hf_processor_mm_kwargs, ) + processed_data.update(passthrough_data) + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/kimi_k25.py b/vllm/model_executor/models/kimi_k25.py index 3054fcc5d8..7f313f44ea 100644 --- a/vllm/model_executor/models/kimi_k25.py +++ b/vllm/model_executor/models/kimi_k25.py @@ -258,16 +258,6 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo]) grid_thws=MultiModalFieldConfig.batched("vision_chunk", keep_on_cpu=True), ) - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - # Override to use the text path instead of token path because vision chunk - # is not considered - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) - def _get_prompt_updates( self, mm_items: MultiModalDataItems, diff --git a/vllm/model_executor/models/kimi_vl.py b/vllm/model_executor/models/kimi_vl.py index e5fa22de7e..fc7533b192 100644 --- a/vllm/model_executor/models/kimi_vl.py +++ b/vllm/model_executor/models/kimi_vl.py @@ -237,6 +237,9 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]): class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]): + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index 85286712fe..e6ae23cca4 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -393,45 +393,38 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]): class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]): - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _postprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, ) -> BatchFeature: - # Text-only input not supported in composite processor - if not (images := mm_data.get("images", [])): - prompt_ids = self.info.get_tokenizer().encode( - prompt, add_special_tokens=False - ) - prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") - - processed_outputs = super()._call_hf_processor( - prompt, - mm_data, - mm_kwargs, - ) + if not mm_data: + return processed_data + images = mm_data.get("images", []) mm_items = self.info.parse_mm_data({"image": images}, validate=False) parsed_images = mm_items.get_items("image", ImageProcessorItems) image_sizes = [ parsed_images.get_image_size(i) for i in range(len(parsed_images)) ] - hf_processor = self.info.get_hf_processor(**mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) num_patches = [ self.info.get_num_patches( image_width=size.width, image_height=size.height, processor=hf_processor, - mm_kwargs=mm_kwargs, + mm_kwargs=hf_processor_mm_kwargs, ) for size in image_sizes ] - processed_outputs["num_patches"] = torch.tensor(num_patches) + processed_data["num_patches"] = torch.tensor(num_patches) - return processed_outputs + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/lightonocr.py b/vllm/model_executor/models/lightonocr.py index 83133e400f..ccb33177fb 100644 --- a/vllm/model_executor/models/lightonocr.py +++ b/vllm/model_executor/models/lightonocr.py @@ -39,20 +39,19 @@ _I = TypeVar("_I", bound=Mistral3ProcessingInfo) class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) - # NOTE: LightOnOCR does not use break/end tokens, so we remove them here. - input_ids = processed_outputs.get("input_ids") + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if not mm_data: + return processed_data + + input_ids = processed_data.get("input_ids") if input_ids is not None: processor = self.info.get_hf_processor() tokenizer = self.info.get_tokenizer() @@ -67,22 +66,22 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn torch.tensor([break_id, end_id]), ) - processed_outputs["input_ids"] = input_ids[keep_mask].unsqueeze(0) - if "attention_mask" in processed_outputs: - processed_outputs["attention_mask"] = processed_outputs[ - "attention_mask" - ][keep_mask].unsqueeze(0) + processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0) + if "attention_mask" in processed_data: + processed_data["attention_mask"] = processed_data["attention_mask"][ + keep_mask + ].unsqueeze(0) # un-pad pixel_values per-image so caches remain independent. - pixel_values = processed_outputs.get("pixel_values") + pixel_values = processed_data.get("pixel_values") if pixel_values is not None: - image_sizes = processed_outputs["image_sizes"] + image_sizes = processed_data["image_sizes"] assert len(pixel_values) == len(image_sizes) - processed_outputs["pixel_values"] = [ + processed_data["pixel_values"] = [ p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes) ] - return processed_outputs + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index 3ddb60452e..c3ec7a4daa 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -321,30 +321,30 @@ class PixtralHFProcessingInfo(BaseLlavaProcessingInfo): class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) - pixel_values = processed_outputs.get("pixel_values") + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if not mm_data: + return processed_data + + pixel_values = processed_data.get("pixel_values") if pixel_values is not None: # Avoid padding since we need the output for each image to be # independent of other images for the cache to work correctly - image_sizes = processed_outputs["image_sizes"] + image_sizes = processed_data["image_sizes"] assert len(pixel_values) == len(image_sizes) - processed_outputs["pixel_values"] = [ + processed_data["pixel_values"] = [ p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes) ] - return processed_outputs + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/llava_next.py b/vllm/model_executor/models/llava_next.py index fec9d6af5e..cf93da2924 100644 --- a/vllm/model_executor/models/llava_next.py +++ b/vllm/model_executor/models/llava_next.py @@ -211,6 +211,9 @@ class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]): class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]): + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/llava_onevision.py b/vllm/model_executor/models/llava_onevision.py index 7b8896f9e0..848372f1f6 100644 --- a/vllm/model_executor/models/llava_onevision.py +++ b/vllm/model_executor/models/llava_onevision.py @@ -321,22 +321,33 @@ class LlavaOnevisionMultiModalProcessor( pixel_values_videos=MultiModalFieldConfig.batched("video"), ) - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + mm_data = dict(mm_data) videos = mm_data.pop("videos", []) assert isinstance(videos, list) if not videos: - return super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=prompt_text, **mm_data), + hf_processor_mm_kwargs, ) + processed_data.update(passthrough_data) + return processed_data # LLaVA-OneVision processor doesn't support multiple videos # with different sizes when converting back to tensors @@ -346,19 +357,13 @@ class LlavaOnevisionMultiModalProcessor( image_token = processor.image_token video_token = processor.video_token - text_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data={}, - mm_kwargs=mm_kwargs, - ) - images = mm_data.pop("images", []) assert isinstance(images, list) if images: - processor_outputs = super()._call_hf_processor( - prompt=image_token * len(images), - mm_data={"images": images}, - mm_kwargs=mm_kwargs, + processor_outputs = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=image_token * len(images), **{"images": images}), + hf_processor_mm_kwargs, ) image_outputs = { k: v @@ -370,10 +375,10 @@ class LlavaOnevisionMultiModalProcessor( pixel_values_videos = [] for video in videos: - item_outputs = super()._call_hf_processor( - prompt=video_token, - mm_data={"videos": video}, - mm_kwargs=mm_kwargs, + item_outputs = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=video_token, **{"videos": video}), + hf_processor_mm_kwargs, ) pixel_values_videos.append(item_outputs["pixel_values_videos"][0]) @@ -381,11 +386,13 @@ class LlavaOnevisionMultiModalProcessor( video_outputs = {"pixel_values_videos": pixel_values_videos} combined_outputs = dict( - text_outputs, + input_ids=prompt_text, **image_outputs, **video_outputs, ) - return BatchFeature(combined_outputs) + processed_data = BatchFeature(combined_outputs) + processed_data.update(passthrough_data) + return processed_data def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/llava_onevision2.py b/vllm/model_executor/models/llava_onevision2.py index be51b1f0f5..5dfb834128 100644 --- a/vllm/model_executor/models/llava_onevision2.py +++ b/vllm/model_executor/models/llava_onevision2.py @@ -286,7 +286,7 @@ def _validate_video_sources(paths, model_config) -> list[str]: # participates in the standard decode-stage pipeline. # # * codec backend: NOT a loader. OV2's codec path needs the video path string -# to survive into ``_call_hf_processor``, where the HF processor builds the +# to survive into ``_apply_hf_processor_main``, where the HF processor builds the # codec canvas + smart_resize + patchify # (pixel_values/image_grid_thw/patch_positions). That transform is # path-level and inseparable; it cannot be reconstructed from pre-decoded @@ -302,7 +302,7 @@ def prepare_codec_video_input(video_path: str) -> tuple: Returns ``(dummy_ndarray, metadata)`` where the ndarray satisfies the parser's 4-D shape check and the metadata carries the actual path to - our ``_call_hf_processor``. Use as:: + our ``_apply_hf_processor_main``. Use as:: multi_modal_data = {"video": prepare_codec_video_input("foo.mp4")} @@ -1283,7 +1283,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo): # ``video_needs_metadata=True`` makes the parser preserve both the # ``(frames, metadata)`` tuples from the frame backend and the # ``(dummy, {marker: path})`` tuples from prepare_codec_video_input; - # both are dispatched by metadata content in ``_call_hf_processor``. + # both are dispatched by metadata content in ``_apply_hf_processor_main``. return LlavaOnevision2MultiModalDataParser( self.get_hf_config().vision_config.spatial_merge_size, video_needs_metadata=True, @@ -1547,18 +1547,23 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser): class LlavaOnevision2MultiModalProcessor( BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo] ): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - # The wrapped OV2 processor is a bare custom class without the standard - # ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path - # fails; overriding this method routes the base class to call us - # directly. - hf_processor = self.info.get_hf_processor(**mm_kwargs) - merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs) + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs) merged_kwargs.setdefault("return_tensors", "pt") call_kwargs = { k: v @@ -1627,21 +1632,25 @@ class LlavaOnevision2MultiModalProcessor( if len(codec_video_paths) > 1 else codec_video_paths[0] ) - # Route through the base ``_call_hf_processor`` so float-tensor + # Route through ``ctx.call_hf_processor`` so float-tensor # dtype postprocessing is applied automatically; inject # ``video_backend="codec"`` via mm_kwargs so the wrapped processor # dispatches to its codec branch. - output = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs={**mm_kwargs, "video_backend": "codec"}, + output = self.info.ctx.call_hf_processor( + self.info.get_hf_processor( + **{**hf_processor_mm_kwargs, "video_backend": "codec"} + ), + dict(text=prompt_text, **mm_data), + {**hf_processor_mm_kwargs, "video_backend": "codec"}, ) data = dict(output) - return BatchFeature( + processed_data = BatchFeature( self._rename_codec_outputs_to_video( data, codec_video_paths, hf_processor ) ) + processed_data.update(passthrough_data) + return processed_data # ---- Frame backend (registered LlavaOnevision2VideoBackend) ------ # Every non-codec video reaches here as a ``(frames_ndarray, metadata)`` @@ -1657,7 +1666,9 @@ class LlavaOnevision2MultiModalProcessor( # local-file gating is enforced by the connector before decoding. if videos_present: timestamp_decimals = int( - mm_kwargs.get("timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS) + hf_processor_mm_kwargs.get( + "timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS + ) ) per_video_frames: list[list[Image.Image]] = [] @@ -1671,7 +1682,7 @@ class LlavaOnevision2MultiModalProcessor( # ``<{t} seconds><|vision_start|><|image_pad|><|vision_end|>`` # blocks (matches the OV2 hf-chat reference exactly). new_prompt = _expand_video_markers_in_prompt( - prompt, + prompt_text, per_video_timestamps, timestamp_decimals=timestamp_decimals, ) @@ -1700,7 +1711,7 @@ class LlavaOnevision2MultiModalProcessor( row_is_video: list[bool] = [] vid_idx = 0 img_idx = 0 - for marker in marker_pattern.finditer(prompt): + for marker in marker_pattern.finditer(prompt_text): if marker.lastgroup == "video": frames = per_video_frames[vid_idx] vid_idx += 1 @@ -1723,15 +1734,15 @@ class LlavaOnevision2MultiModalProcessor( merged_mm_data.pop("videos", None) merged_mm_data["images"] = flat_frames - # Route through the base ``_call_hf_processor`` (applies float-tensor + # Route through ``ctx.call_hf_processor`` (applies float-tensor # dtype postprocessing automatically). The wrapped processor's image # branch ignores video/codec-only kwargs and does not forward extra # **kwargs to the image processor, so passing the full merged kwarg # set here is a no-op beyond return_tensors/padding. - output = super()._call_hf_processor( - prompt=new_prompt, - mm_data=merged_mm_data, - mm_kwargs=mm_kwargs, + output = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=new_prompt, **merged_mm_data), + hf_processor_mm_kwargs, ) data = dict(output) @@ -1788,17 +1799,21 @@ class LlavaOnevision2MultiModalProcessor( (len(per_video_timestamps),), dtype=torch.long ) - return BatchFeature(data) + processed_data = BatchFeature(data) + processed_data.update(passthrough_data) + return processed_data # ---- Image-only / text-only call -------------------------------- - # No videos present: delegate to the base ``_call_hf_processor``, which + # No videos present: delegate to ``ctx.call_hf_processor``, which # runs the wrapped processor over the (possibly empty) image set and # applies float-tensor dtype postprocessing automatically. - return super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=prompt_text, **mm_data), + hf_processor_mm_kwargs, ) + processed_data.update(passthrough_data) + return processed_data def _rename_codec_outputs_to_video( self, diff --git a/vllm/model_executor/models/midashenglm.py b/vllm/model_executor/models/midashenglm.py index 8ddfd05f72..9464c3f248 100644 --- a/vllm/model_executor/models/midashenglm.py +++ b/vllm/model_executor/models/midashenglm.py @@ -26,7 +26,7 @@ import collections import collections.abc from collections.abc import Callable, Iterable, Mapping, Sequence -from typing import Annotated, Any, TypeAlias, cast +from typing import Annotated, TypeAlias, cast import numpy as np import torch @@ -583,15 +583,17 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing class MiDashengLMMultiModalProcessor( BaseMultiModalProcessor[MiDashengLMProcessingInfo] ): - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _preprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, Any], - ) -> BatchFeature: + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + mm_data = dict(mm_data) audios = mm_data.pop("audios", []) - # + Padding min_audio_len = self.info.get_min_audio_len() processed_audios = [ np.pad( @@ -608,20 +610,7 @@ class MiDashengLMMultiModalProcessor( if processed_audios: mm_data["audio"] = processed_audios - if not mm_data.get("audio", []): - prompt_ids = self.info.get_tokenizer().encode(prompt) - prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") - - mm_kwargs = dict( - **mm_kwargs, - ) - - return super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) + return mm_data, hf_processor_mm_kwargs def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/mimo_v2_omni.py b/vllm/model_executor/models/mimo_v2_omni.py index 8f78409581..bd418d9c09 100644 --- a/vllm/model_executor/models/mimo_v2_omni.py +++ b/vllm/model_executor/models/mimo_v2_omni.py @@ -912,16 +912,23 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio") return fields - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: """Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo. Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses the singular form. """ + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + # Remap audios → audio (MiMoOmniProcessor uses singular param name) if "audios" in mm_data: mm_data = {**mm_data, "audio": mm_data["audios"]} @@ -999,7 +1006,13 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing mm_data = {**mm_data, "videos": converted} - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + mm_data, + hf_processor_mm_kwargs, + ) + processed_data.update(passthrough_data) + return processed_data def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/minicpmo.py b/vllm/model_executor/models/minicpmo.py index 3dd3f8fd3b..527117bd0b 100644 --- a/vllm/model_executor/models/minicpmo.py +++ b/vllm/model_executor/models/minicpmo.py @@ -448,7 +448,7 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems): audio_inputs = {} else: - audio_inputs = self._base_call_hf_processor( + audio_inputs = self._call_hf_processor_on_prompts( prompts=[self.info.audio_pattern] * len(parsed_audios), mm_data={"audios": [[audio] for audio in parsed_audios]}, mm_kwargs={**mm_kwargs, "chunk_input": True}, diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index 968490d4e9..b638e1034d 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -881,7 +881,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): if isinstance(parsed_images, MiniCPMVImageEmbeddingItems): image_inputs = {} else: - image_inputs = self._base_call_hf_processor( + image_inputs = self._call_hf_processor_on_prompts( prompts=[self.info.image_pattern] * len(parsed_images), mm_data={"images": [[image] for image in parsed_images]}, mm_kwargs=mm_kwargs, @@ -906,7 +906,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems): video_inputs = {} else: - video_inputs = self._base_call_hf_processor( + video_inputs = self._call_hf_processor_on_prompts( prompts=[ self.info.image_pattern * len(video) for video in parsed_videos ], @@ -983,7 +983,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): return new_token_ids, placeholders - def _base_call_hf_processor( + def _call_hf_processor_on_prompts( self, prompts: list[str], mm_data: Mapping[str, Sequence[object]], @@ -993,19 +993,19 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): ) -> dict[str, NestedTensors]: # This processor supports zipping prompt and mm_data together if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}: - inputs = super()._call_hf_processor( - prompt=prompts, # type: ignore - mm_data=mm_data, - mm_kwargs=mm_kwargs, + inputs = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**mm_kwargs), + dict(text=prompts, **mm_data), + mm_kwargs, ) else: inputs = defaultdict[str, list[torch.Tensor]](list) for i, prompt in enumerate(prompts): - inputs_one = super()._call_hf_processor( - prompt=prompt, - mm_data={k: v[i] for k, v in mm_data.items()}, - mm_kwargs=mm_kwargs, + inputs_one = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**mm_kwargs), + dict(text=prompt, **{k: v[i] for k, v in mm_data.items()}), + mm_kwargs, ) for k, v in inputs_one.items(): @@ -1014,23 +1014,31 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): return {k: inputs[k] for k in out_keys} - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + tokenizer = self.info.get_tokenizer() - input_ids = torch.tensor([tokenizer.encode(prompt)]) - mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs) + input_ids = torch.tensor([tokenizer.encode(prompt_text)]) + mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs) - return BatchFeature( + processed_data = BatchFeature( { "input_ids": input_ids, **mm_inputs, } ) + processed_data.update(passthrough_data) + return processed_data def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index 22adf1c484..11f214c65b 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -233,30 +233,30 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo]) class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) - pixel_values = processed_outputs.get("pixel_values") + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if not mm_data: + return processed_data + + pixel_values = processed_data.get("pixel_values") if pixel_values is not None: # Avoid padding since we need the output for each image to be # independent of other images for the cache to work correctly - image_sizes = processed_outputs["image_sizes"] + image_sizes = processed_data["image_sizes"] assert len(pixel_values) == len(image_sizes) - processed_outputs["pixel_values"] = [ + processed_data["pixel_values"] = [ p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes) ] - return processed_outputs + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index ca862596ef..9860996bcf 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -589,23 +589,23 @@ class Mllama4ProcessingInfo(BaseProcessingInfo): class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) - processor = self.info.get_hf_processor(**mm_kwargs) + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if not mm_data: + return processed_data + + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) image_processor = processor.image_processor vision_config = self.info.get_hf_config().vision_config - if processed_outputs.get("pixel_values") is not None: + if processed_data.get("pixel_values") is not None: assert "images" in mm_data, ( "images expected to be in mm_data when pixel_values is present" ) @@ -636,10 +636,10 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]) 1 if r_h * r_w == 1 else 1 + r_h * r_w for (r_h, r_w) in aspect_ratios ] - processed_outputs["aspect_ratios"] = torch.tensor(aspect_ratios) - processed_outputs["patches_per_image"] = torch.tensor(patches_per_image) + processed_data["aspect_ratios"] = torch.tensor(aspect_ratios) + processed_data["patches_per_image"] = torch.tensor(patches_per_image) - return processed_outputs + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index 2b963e339a..b823d9c090 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -1123,17 +1123,44 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]): class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]): - def _call_hf_processor( + def _postprocess_prompt(self, prompt: list[int]) -> list[int]: + processor = self.info.get_hf_processor() + + # The chat template is already applied to the prompt tokens + # Use message_format="none" to avoid applying it again + # Prepend an empty space if `always_start_with_space` is True + tokens = processor.get_tokens_input( + self.info.get_tokenizer().decode(prompt), + message_format="none", + always_start_with_space=True, + ) + + # Prepend a BOS token id to the tokens + return self.info.ctx.call_hf_processor( + processor.process, + dict(tokens=tokens), + )["input_ids"].tolist() + + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - hf_processor = self.info.get_hf_processor(**mm_kwargs) - processed_outputs = self.info.ctx.call_hf_processor( + mm_counts = mm_items.get_all_counts() + + valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0}) + processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not processor_data: + return BatchFeature(dict(passthrough_data)) + + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + prompt_text = self.dummy_inputs.get_dummy_text(mm_counts) + + processed_data = self.info.ctx.call_hf_processor( hf_processor.process, - dict(text=prompt, **mm_data), - mm_kwargs, + dict(text=prompt_text, **processor_data), + hf_processor_mm_kwargs, ) tokenizer = hf_processor.tokenizer @@ -1141,17 +1168,16 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]): image_processor = hf_processor.image_processor - input_ids: torch.Tensor = processed_outputs.pop("input_ids") - processed_outputs["input_ids"] = input_ids.unsqueeze(0) + processed_data.pop("input_ids") - if (images := mm_data.get("images")) is not None: + if (images := processor_data.get("images")) is not None: mm_items = self.info.parse_mm_data({"image": images}, validate=False) parsed_images = mm_items.get_items("image", ImageProcessorItems) image_sizes = [ parsed_images.get_image_size(i) for i in range(len(parsed_images)) ] - feat_is_patch = processed_outputs["image_input_idx"] >= 0 + feat_is_patch = processed_data["image_input_idx"] >= 0 tilings = [ self.info.select_tiling( @@ -1165,35 +1191,12 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]): num_crops = torch.tensor(tilings).prod(-1) + 1 assert num_crops.sum() == len(feat_is_patch) - processed_outputs["num_crops"] = num_crops - processed_outputs["img_patch_id"] = image_patch_id + processed_data["num_crops"] = num_crops + processed_data["img_patch_id"] = image_patch_id - return processed_outputs + processed_data.update(passthrough_data) - def _apply_hf_processor_tokens_only( - self, - prompt_tokens: list[int], - ) -> list[int]: - processor = self.info.get_hf_processor() - - # The chat template is already applied to the prompt tokens - # Use message_format="none" to avoid applying it again - # Prepend an empty space if `always_start_with_space` is True - tokens = processor.get_tokens_input( - self.info.get_tokenizer().decode(prompt_tokens), - message_format="none", - always_start_with_space=True, - ) - - # Prepend a BOS token id to the tokens - processed_data = self.info.ctx.call_hf_processor( - processor.process, - dict(tokens=tokens), - ) - prompt_ids = processed_data.pop("input_ids").tolist() - print(prompt_ids, len(prompt_ids)) - - return prompt_ids + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/molmo2.py b/vllm/model_executor/models/molmo2.py index 861e31928b..d73160d689 100644 --- a/vllm/model_executor/models/molmo2.py +++ b/vllm/model_executor/models/molmo2.py @@ -1929,30 +1929,33 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]): class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): - def _apply_hf_processor_tokens_only( - self, - prompt_tokens: list[int], - ) -> list[int]: + def _postprocess_prompt(self, prompt: list[int]) -> list[int]: processor = self.info.get_hf_processor() tokenizer = processor.tokenizer bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id - if len(prompt_tokens) == 0 or prompt_tokens[0] != bos_token_id: + if len(prompt) == 0 or prompt[0] != bos_token_id: # Prepend the bos token to the prompt tokens - prompt_tokens = [bos_token_id] + prompt_tokens + prompt = [bos_token_id] + prompt - return prompt_tokens + return prompt - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - mm_data = dict(mm_data) + mm_counts = mm_items.get_all_counts() + + valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0}) + processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_counts) + + mm_data = dict(processor_data) hf_config = self.info.get_hf_config() - hf_processor = self.info.get_hf_processor(**mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature: res = hf_processor(text=text, images=images, videos=videos, **kwargs) @@ -1984,14 +1987,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): # NOTE: metadata.frames_indices indicates # the sampled frames indices of pre-sampled videos, which is # used to calculate the timestamps. Make sure that - # do_sample_frames in mm_kwargs is false for presampled videos. + # do_sample_frames in hf_processor_mm_kwargs is false for + # presampled videos. - # NOTE: a copy of mm_kwargs is created to update do_sample_frames, - # otherwise mm_hash for the object will be incorrect. - video_mm_kwargs = dict(**mm_kwargs) + # NOTE: a copy of hf_processor_mm_kwargs is created to update + # do_sample_frames, otherwise mm_hash for the object will be + # incorrect. + video_mm_kwargs = dict(**hf_processor_mm_kwargs) if "do_sample_frames" not in video_mm_kwargs: # molmo_utils already has "do_sample_frames" in - # mm_kwargs, don't overwrite it. + # hf_processor_mm_kwargs, don't overwrite it. video_mm_kwargs["do_sample_frames"] = metadata.get( "do_sample_frames", False ) @@ -2015,7 +2020,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): input_ids = input_ids[:, 1:] video_string = tokenizer.batch_decode(input_ids)[0] - prompt = prompt.replace(VIDEO_PROMPT, video_string, 1) + prompt_text = prompt_text.replace(VIDEO_PROMPT, video_string, 1) video_grids = video_outputs.pop("video_grids") assert video_grids[:, 0].sum() == len( @@ -2054,10 +2059,10 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): else: all_video_outputs = dict() - processed_outputs = self.info.ctx.call_hf_processor( + processed_data = self.info.ctx.call_hf_processor( patched_call, - dict(text=prompt, **mm_data), - mm_kwargs, + dict(text=prompt_text, **mm_data), + hf_processor_mm_kwargs, ) if (images := mm_data.get("images")) is not None: @@ -2077,22 +2082,22 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): for image_size in image_sizes ] num_crops = torch.tensor(tilings).prod(-1) + 1 - assert sum(num_crops) == len(processed_outputs["pixel_values"]) - assert sum(num_crops) == processed_outputs["image_num_crops"].sum().item() + assert sum(num_crops) == len(processed_data["pixel_values"]) + assert sum(num_crops) == processed_data["image_num_crops"].sum().item() - image_grids = processed_outputs.pop("image_grids") + image_grids = processed_data.pop("image_grids") image_num_pooled_patches = image_grids[:, :2].prod(dim=1) + image_grids[ :, 2: ].prod(dim=1) - processed_outputs["image_num_pooled_patches"] = image_num_pooled_patches - n_patches = processed_outputs["pixel_values"].shape[1] - processed_outputs["image_num_patches"] = ( - processed_outputs["image_num_crops"] * n_patches + processed_data["image_num_pooled_patches"] = image_num_pooled_patches + n_patches = processed_data["pixel_values"].shape[1] + processed_data["image_num_patches"] = ( + processed_data["image_num_crops"] * n_patches ) ( - processed_outputs["image_tokens"], - processed_outputs["num_image_tokens"], + processed_data["image_tokens"], + processed_data["num_image_tokens"], ) = build_flat_image_bool_length( image_grids, hf_config, @@ -2101,7 +2106,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): use_single_crop_start_token=hf_processor.use_single_crop_start_token, ) - return BatchFeature({**processed_outputs, **all_video_outputs}) + processed_data.update(all_video_outputs) + processed_data.update(passthrough_data) + processed_data.pop("input_ids") + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/moondream3.py b/vllm/model_executor/models/moondream3.py index 0464150bd4..45833f6241 100644 --- a/vllm/model_executor/models/moondream3.py +++ b/vllm/model_executor/models/moondream3.py @@ -942,16 +942,6 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing image_placeholder: str = "" bos_image_placeholder: str = "<|endoftext|>" - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - # Moondream3's processor handles images directly rather than exposing a - # separate `image_processor`, so keep the cache path on text+MM calls. - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) - @cached_property def bos_image_placeholder_tokens(self) -> list[int]: tokenizer = self.info.get_tokenizer() diff --git a/vllm/model_executor/models/moss_audio.py b/vllm/model_executor/models/moss_audio.py index ceed6245a1..7c4fa3b8f8 100644 --- a/vllm/model_executor/models/moss_audio.py +++ b/vllm/model_executor/models/moss_audio.py @@ -1335,23 +1335,32 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + mm_data = dict(mm_data) audios = mm_data.pop("audios", []) if audios: mm_data["audio"] = audios - mm_kwargs = dict(mm_kwargs) - processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs) - return self.info.ctx.call_hf_processor( + hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs) + processor_kwargs = _filter_moss_audio_processor_config(hf_processor_mm_kwargs) + processed_data = self.info.ctx.call_hf_processor( self.info.get_hf_processor(**processor_kwargs), - dict(text=prompt, **mm_data), + mm_data, {}, ) + processed_data.update(passthrough_data) + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/moss_transcribe_diarize.py b/vllm/model_executor/models/moss_transcribe_diarize.py index 421901d0ee..8122308af4 100644 --- a/vllm/model_executor/models/moss_transcribe_diarize.py +++ b/vllm/model_executor/models/moss_transcribe_diarize.py @@ -461,24 +461,29 @@ class MossTranscribeDiarizeDummyInputsBuilder( class MossTranscribeDiarizeMultiModalProcessor( BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo] ): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - tokenizer = self.info.get_tokenizer() - audios = _get_audios_from_mm_data(mm_data) - if not audios: - input_ids = tokenizer.encode(prompt, add_special_tokens=False) - return BatchFeature({"input_ids": [input_ids]}, tensor_type="pt") + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) - processed = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**mm_kwargs), - dict(text=prompt, audio=audios), - mm_kwargs, - ) - return _add_vllm_audio_metadata(processed, len(audios)) + def _preprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + audios = _get_audios_from_mm_data(mm_data) + + return dict(audio=audios), hf_processor_mm_kwargs + + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + audios = _get_audios_from_mm_data(mm_data) + if audios: + processed_data = _add_vllm_audio_metadata(processed_data, len(audios)) + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/muse_glimmer.py b/vllm/model_executor/models/muse_glimmer.py index fcdfb82333..a0281ec0aa 100644 --- a/vllm/model_executor/models/muse_glimmer.py +++ b/vllm/model_executor/models/muse_glimmer.py @@ -262,13 +262,19 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing class MuseGlimmerMultiModalProcessor( BaseMultiModalProcessor[MuseGlimmerProcessingInfo] ): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - processor = self.info.get_hf_processor(**mm_kwargs) + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) tokenizer = processor.tokenizer config = self.info.get_hf_config() images = mm_data.get("images", ()) @@ -276,7 +282,7 @@ class MuseGlimmerMultiModalProcessor( if not isinstance(images, Sequence) or not isinstance(videos, Sequence): raise TypeError("MuseGlimmer multi-modal data must be a sequence") - prompt_ids = tokenizer.encode(prompt, add_special_tokens=False) + prompt_ids = tokenizer.encode(prompt_text, add_special_tokens=False) image_sentinel_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN) video_sentinel_id = tokenizer.convert_tokens_to_ids(VIDEO_TOKEN) if prompt_ids.count(image_sentinel_id) != len(images): @@ -354,7 +360,9 @@ class MuseGlimmerMultiModalProcessor( video_pixel_values=video_pixels, video_feature_sizes=torch.tensor(video_sizes), ) - return BatchFeature(data=data, tensor_type=None) + processed_data = BatchFeature(data=data, tensor_type=None) + processed_data.update(passthrough_data) + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 679495e3ec..0e9eda5c82 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -361,18 +361,8 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo): class NanoNemotronVLMultiModalProcessor( BaseMultiModalProcessor[NanoNemotronVLProcessingInfo] ): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - """ - Bypass `call_hf_processor_mm_only` by no-op overriding`_call_hf_processor`, - so it chooses this path: - `type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor` - """ - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) def _get_image_fields_config(self, hf_inputs: BatchFeature): if self.info.is_dynamic_tiler: @@ -406,7 +396,9 @@ class NanoNemotronVLMultiModalProcessor( ) def _get_audio_fields_config(self, hf_inputs: BatchFeature): - audio_num_clips = torch.as_tensor(hf_inputs["audio_num_clips"]) + audio_num_clips = torch.as_tensor( + hf_inputs.get("audio_num_clips", torch.empty(0)) + ) return dict( input_audio_features=MultiModalFieldConfig.flat_from_sizes( @@ -733,11 +725,8 @@ class NanoNemotronVLMultiModalProcessor( # Bypass the cached path: the HF processor must receive the # prompt (with injected ) and the audio data # together so it can perform audio-token replacement natively. - ( - prompt_ids, - mm_info, - is_update_applied, - ) = self._apply_hf_processor(inputs, timing_ctx) + mm_info = self._apply_hf_processor(inputs, timing_ctx) + prompt_ids = self._postprocess_prompt(inputs.prompt) with timing_ctx.record("apply_prompt_updates"): prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates( @@ -745,7 +734,7 @@ class NanoNemotronVLMultiModalProcessor( prompt_ids=prompt_ids, mm_kwargs=mm_info.kwargs, mm_prompt_updates=mm_info.prompt_updates, - is_update_applied=is_update_applied, + is_update_applied=self.hf_processor_applies_updates, ) mm_placeholder_ranges = { diff --git a/vllm/model_executor/models/nemotron_parse.py b/vllm/model_executor/models/nemotron_parse.py index 9ce8ba2246..12c7b58048 100644 --- a/vllm/model_executor/models/nemotron_parse.py +++ b/vllm/model_executor/models/nemotron_parse.py @@ -411,22 +411,6 @@ class NemotronParseMultiModalProcessor( ) -> list[int]: return [0] - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - if mm_data: - processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs) - else: - hf_processor = self.info.get_hf_processor() - tokenizer = hf_processor.tokenizer - processed_outputs = tokenizer( - prompt, add_special_tokens=False, return_tensors="pt" - ) - return processed_outputs - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/ovis.py b/vllm/model_executor/models/ovis.py index 4d023f51e3..d1a805f608 100644 --- a/vllm/model_executor/models/ovis.py +++ b/vllm/model_executor/models/ovis.py @@ -341,41 +341,30 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]): # -300 is image_atom token, filter them out return [vte_vocab_size + x + 300 for x in image_indicators if x < -300] - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _postprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, ) -> BatchFeature: if not mm_data: - # Avoid warning from HF logger for text-only input - tokenizer = self.info.get_tokenizer() - prompt_ids = tokenizer.encode(prompt, add_special_tokens=False) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") - - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) + return processed_data hf_processor = self.info.get_hf_processor() image_indicators = [ hf_processor.construct_image_indicators(grid) - for grid in processed_outputs["grids"] + for grid in processed_data["grids"] ] indicator_tokens = [ self.image_indicators_to_visual_tokens(indicator) for indicator in image_indicators ] - processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens) - return processed_outputs + processed_data["indicator_tokens"] = torch.tensor(indicator_tokens) - def _apply_hf_processor_tokens_only( - self, - prompt_tokens: list[int], - ) -> list[int]: - return prompt_tokens + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py index 1ccf820e0f..c55f480537 100644 --- a/vllm/model_executor/models/ovis2_5.py +++ b/vllm/model_executor/models/ovis2_5.py @@ -332,53 +332,43 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]) if x >= INDICATOR_IDS[0] ] - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _postprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, ) -> BatchFeature: if not mm_data: - # Avoid warning from HF logger for text-only input - tokenizer = self.info.get_tokenizer() - prompt_ids = tokenizer.encode(prompt, add_special_tokens=False) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") + return processed_data - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) hf_processor = self.info.get_hf_processor() if "videos" in mm_data: visual_indicators = [ hf_processor.construct_visual_indicators((1, 1, 1), True) - for grid in processed_outputs["video_grids"] + for grid in processed_data["video_grids"] ] indicator_tokens = [ self.visual_indicators_to_visual_tokens(indicator) for indicator in visual_indicators ] - processed_outputs["video_indicator_tokens"] = torch.tensor(indicator_tokens) + processed_data["video_indicator_tokens"] = torch.tensor(indicator_tokens) if "images" in mm_data: visual_indicators = [ hf_processor.construct_visual_indicators((1, 1, 1), False) - for grid in processed_outputs["grids"] + for grid in processed_data["grids"] ] indicator_tokens = [ self.visual_indicators_to_visual_tokens(indicator) for indicator in visual_indicators ] - processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens) - return processed_outputs + processed_data["indicator_tokens"] = torch.tensor(indicator_tokens) - def _apply_hf_processor_tokens_only( - self, - prompt_tokens: list[int], - ) -> list[int]: - return prompt_tokens + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/paddleocr_vl.py b/vllm/model_executor/models/paddleocr_vl.py index 61f1ed9d83..a432a8d5f0 100644 --- a/vllm/model_executor/models/paddleocr_vl.py +++ b/vllm/model_executor/models/paddleocr_vl.py @@ -244,32 +244,37 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing class PaddleOCRVLMultiModalProcessor( BaseMultiModalProcessor[PaddleOCRVLProcessingInfo] ): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - if mm_data: - final_mm_kwargs = dict(mm_kwargs or {}) - final_mm_kwargs.setdefault("images_kwargs", {}) - # vLLM use PIL.Image, always set channel_last - final_mm_kwargs["input_data_format"] = ChannelDimension.LAST - processed_outputs = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**final_mm_kwargs), - dict(text=prompt, **mm_data), - mm_kwargs, - ) - num_patches_per_image = processed_outputs["image_grid_thw"].prod(-1) - processed_outputs["pixel_values"] = processed_outputs["pixel_values"].split( - num_patches_per_image.tolist() - ) - else: - tokenizer = self.info.get_tokenizer() - processed_outputs = tokenizer( - prompt, add_special_tokens=True, return_tensors="pt" - ) - return processed_outputs + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + + final_mm_kwargs = dict(hf_processor_mm_kwargs or {}) + final_mm_kwargs.setdefault("images_kwargs", {}) + # vLLM use PIL.Image, always set channel_last + final_mm_kwargs["input_data_format"] = ChannelDimension.LAST + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**final_mm_kwargs), + dict(text=prompt_text, **mm_data), + hf_processor_mm_kwargs, + ) + num_patches_per_image = processed_data["image_grid_thw"].prod(-1) + processed_data["pixel_values"] = processed_data["pixel_values"].split( + num_patches_per_image.tolist() + ) + processed_data.update(passthrough_data) + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/paligemma.py b/vllm/model_executor/models/paligemma.py index 1e80e8d818..781e9a620d 100644 --- a/vllm/model_executor/models/paligemma.py +++ b/vllm/model_executor/models/paligemma.py @@ -152,23 +152,6 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - tokenizer = self.info.get_tokenizer() - if not mm_data: - prompt_ids = tokenizer.encode(prompt, add_special_tokens=False) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") - - return super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/phi3v.py b/vllm/model_executor/models/phi3v.py index aca1b9a632..5ed955db82 100644 --- a/vllm/model_executor/models/phi3v.py +++ b/vllm/model_executor/models/phi3v.py @@ -396,19 +396,19 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]): class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) - input_ids = processed_outputs["input_ids"] + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if not mm_data: + return processed_data + + input_ids = processed_data["input_ids"] assert isinstance(input_ids, torch.Tensor) # Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids, @@ -416,7 +416,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]): # Therefore, we need to do an early replacement here input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID) - return processed_outputs + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/phi4mm.py b/vllm/model_executor/models/phi4mm.py index b18ff6bb0f..4dc5d408a4 100644 --- a/vllm/model_executor/models/phi4mm.py +++ b/vllm/model_executor/models/phi4mm.py @@ -868,43 +868,52 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]): class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - if not mm_data: - prompt_ids = self.info.get_tokenizer().encode(prompt) - prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - sr = self.info.get_feature_extractor(**mm_kwargs).sampling_rate + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + + sr = self.info.get_feature_extractor(**hf_processor_mm_kwargs).sampling_rate if audio_data := mm_data.get("audios", []): mm_data["audios"] = [(data, sr) for data in audio_data] - processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs) + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=prompt_text, **mm_data), + hf_processor_mm_kwargs, + ) - hf_processor = self.info.get_hf_processor(**mm_kwargs) + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) num_img_tokens = [ self.info.get_num_image_tokens( image_width=img_size[0], image_height=img_size[1], processor=hf_processor, ) - for img_size in processed_outputs["image_sizes"] + for img_size in processed_data["image_sizes"] ] - processed_outputs["num_img_tokens"] = num_img_tokens + processed_data["num_img_tokens"] = num_img_tokens - audio_features = processed_outputs["input_audio_embeds"] + audio_features = processed_data["input_audio_embeds"] feature_sizes = [ self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data ] - processed_outputs["input_audio_embeds"] = [ + processed_data["input_audio_embeds"] = [ audio_features[idx, :size] for idx, size in enumerate(feature_sizes) ] + processed_data.update(passthrough_data) - return processed_outputs + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/phi4siglip.py b/vllm/model_executor/models/phi4siglip.py index cb43161486..6c6c6efa2d 100644 --- a/vllm/model_executor/models/phi4siglip.py +++ b/vllm/model_executor/models/phi4siglip.py @@ -142,16 +142,22 @@ class Phi4SiglipDummyInputsBuilder( class Phi4SiglipMultiModalProcessor( BaseMultiModalProcessor[Phi4SiglipProcessingInfo], ): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - processed = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + + processed = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=prompt_text, **mm_data), + hf_processor_mm_kwargs, ) # The HF processor's tokenizer_image_token() replaces the "" @@ -163,10 +169,12 @@ class Phi4SiglipMultiModalProcessor( # NOTE: tokenizer.__call__() (not .encode()) must be used so that # added/special tokens like <|user|>, <|end|> are kept as single IDs. tokenizer = self.info.get_tokenizer() - new_ids = tokenizer(prompt).input_ids + new_ids = tokenizer(prompt_text).input_ids processed["input_ids"] = torch.tensor([new_ids]) - return processed + processed_data = processed + processed_data.update(passthrough_data) + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index 48b3310d30..a24d2484ab 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -51,12 +51,10 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, - MultiModalProcessingInfo, ProcessorInputs, PromptReplacement, PromptUpdate, PromptUpdateDetails, - TimingContext, ) from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors @@ -213,6 +211,9 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]): class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]): + # The tokens are already inserted by the chat template + hf_processor_applies_updates = True + def _get_mm_fields_config( self, hf_inputs: Mapping[str, NestedTensors], @@ -220,24 +221,35 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]) ) -> Mapping[str, MultiModalFieldConfig]: return dict(images=MultiModalFieldConfig.batched("image")) - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + outputs = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**mm_kwargs), - dict(text=prompt, **mm_data), + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=prompt_text, **mm_data), # Avoid padding issue - dict(**mm_kwargs, return_tensors=None), + dict(**hf_processor_mm_kwargs, return_tensors=None), ) # Missing batch dimension if is_list_of(outputs["input_ids"], int): outputs["input_ids"] = [outputs["input_ids"]] - return outputs + processed_data = outputs + processed_data.update(passthrough_data) + return processed_data def _get_prompt_updates( self, @@ -273,16 +285,6 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]) ), ] - def _cached_apply_hf_processor( - self, - inputs: ProcessorInputs, - timing_ctx: TimingContext, - ) -> tuple[list[int], MultiModalProcessingInfo, bool]: - prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx) - - # NOTE: The tokens are already inserted by the chat template - return prompt_ids, mm_info, True - @MULTIMODAL_REGISTRY.register_processor( PixtralMultiModalProcessor, diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index 052b2823de..1f335cfc24 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -499,83 +499,6 @@ class Qwen2_5OmniThinkerDummyInputsBuilder( class Qwen2_5OmniThinkerMultiModalProcessor( BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo] ): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - mm_data = dict(mm_data) - audios = mm_data.pop("audios", []) - - # NOTE: WhisperFeatureExtractor cannot handle empty list of audios - if audios: - # NOTE: Qwen2.5-Omni processor accept "audio" - mm_data["audio"] = audios - mm_kwargs = dict( - **mm_kwargs, - ) - elif ( - mm_kwargs.get("use_audio_in_video") - and "audio" not in mm_data - and mm_data.get("videos") - ): - # A subclass (e.g. Qwen3-Omni) may have already popped "audios" - # and populated mm_data["audio"] itself before delegating here, - # in which case this isn't a genuine "no audio" case. Likewise, - # mm_data can be empty on a multimodal-processor-cache hit, where - # there's nothing to (re-)process this call and the real result - # comes from the cache — not a genuine "no audio" case either. - raise ValueError( - "Video doesn't have audio track with `audio_in_video=True`" - ) - - merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs) - if mm_data.get("videos") and ( - merged.keys() & {"size", "min_pixels", "max_pixels"} - ): - mm_kwargs = dict(mm_kwargs) - video_size = dict(self.info.get_hf_processor().video_processor.size) - size_override = merged.get("size") - if size_override is not None: - video_size = video_size | size_override - min_pixels = merged.get("min_pixels") - if min_pixels is not None: - video_size["shortest_edge"] = min_pixels - max_pixels = merged.get("max_pixels") - if max_pixels is not None: - video_size["longest_edge"] = max_pixels - mm_kwargs["size"] = video_size - - hf_inputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) - - input_features = hf_inputs.pop("input_features", None) - feature_attention_mask = hf_inputs.get("feature_attention_mask", None) - if "input_audio_features" not in hf_inputs and input_features is not None: - if feature_attention_mask is not None: - input_features = input_features.permute(0, 2, 1)[ - feature_attention_mask.bool() - ].permute(1, 0) - hf_inputs["input_audio_features"] = input_features - if ( - "audio_feature_lengths" not in hf_inputs - and feature_attention_mask is not None - ): - hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1) - - video_second_per_grid = hf_inputs.get("video_second_per_grid", None) - if video_second_per_grid is not None: - hf_inputs["second_per_grid_ts"] = video_second_per_grid - - use_audio_in_video = mm_kwargs.get("use_audio_in_video", False) - hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video) - - return hf_inputs - def _get_mm_fields_config( self, hf_inputs: BatchFeature, @@ -885,7 +808,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor( ), ] - def _apply_hf_processor_mm_only( + def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, hf_processor_mm_kwargs: Mapping[str, object], @@ -903,13 +826,87 @@ class Qwen2_5OmniThinkerMultiModalProcessor( ) mm_counts["audio"] -= mm_counts["video"] - _, mm_processed_data = self._apply_hf_processor_text_mm( - prompt_text=self.dummy_inputs.get_dummy_text(mm_counts), - mm_items=mm_items, - hf_processor_mm_kwargs=hf_processor_mm_kwargs, + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_counts) + + mm_data = dict(processor_data) + audios = mm_data.pop("audios", []) + + # NOTE: WhisperFeatureExtractor cannot handle empty list of audios + if audios: + # NOTE: Qwen2.5-Omni processor accept "audio" + mm_data["audio"] = audios + hf_processor_mm_kwargs = dict( + **hf_processor_mm_kwargs, + ) + elif ( + hf_processor_mm_kwargs.get("use_audio_in_video") + and "audio" not in mm_data + and mm_data.get("videos") + ): + # A subclass (e.g. Qwen3-Omni) may have already popped "audios" + # and populated mm_data["audio"] itself before delegating here, + # in which case this isn't a genuine "no audio" case. Likewise, + # mm_data can be empty on a multimodal-processor-cache hit, where + # there's nothing to (re-)process this call and the real result + # comes from the cache — not a genuine "no audio" case either. + raise ValueError( + "Video doesn't have audio track with `audio_in_video=True`" + ) + + merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs) + if mm_data.get("videos") and ( + merged.keys() & {"size", "min_pixels", "max_pixels"} + ): + hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs) + video_size = dict(self.info.get_hf_processor().video_processor.size) + size_override = merged.get("size") + if size_override is not None: + video_size = video_size | size_override + min_pixels = merged.get("min_pixels") + if min_pixels is not None: + video_size["shortest_edge"] = min_pixels + max_pixels = merged.get("max_pixels") + if max_pixels is not None: + video_size["longest_edge"] = max_pixels + hf_processor_mm_kwargs["size"] = video_size + + hf_inputs = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=prompt_text, **mm_data), + hf_processor_mm_kwargs, ) - return mm_processed_data + input_features = hf_inputs.pop("input_features", None) + feature_attention_mask = hf_inputs.get("feature_attention_mask", None) + if "input_audio_features" not in hf_inputs and input_features is not None: + if feature_attention_mask is not None: + input_features = input_features.permute(0, 2, 1)[ + feature_attention_mask.bool() + ].permute(1, 0) + hf_inputs["input_audio_features"] = input_features + if ( + "audio_feature_lengths" not in hf_inputs + and feature_attention_mask is not None + ): + hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1) + + video_second_per_grid = hf_inputs.get("video_second_per_grid", None) + if video_second_per_grid is not None: + hf_inputs["second_per_grid_ts"] = video_second_per_grid + + use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False) + hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video) + + processed_data = hf_inputs + processed_data.update(passthrough_data) + processed_data.pop("input_ids") + + return processed_data class Qwen2_5OmniConditionalGenerationMixin: diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 396a892cab..3e5deaad91 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -1154,16 +1154,6 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor): second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True), ) - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - # Override to use the text path instead of token path to use the - # video-specific logic in processing_qwen2_5_vl.py - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) - def _get_prompt_updates( self, mm_items: MultiModalDataItems, diff --git a/vllm/model_executor/models/qwen2_audio.py b/vllm/model_executor/models/qwen2_audio.py index 695d483355..74341575e7 100644 --- a/vllm/model_executor/models/qwen2_audio.py +++ b/vllm/model_executor/models/qwen2_audio.py @@ -235,36 +235,27 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]): - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _preprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, Any], - ) -> BatchFeature: - # NOTE - we rename audios -> audio in mm data because transformers has - # deprecated audios for the qwen2audio processor and will remove - # support for it in transformers 4.54. + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + + mm_data = dict(mm_data) audios = mm_data.pop("audios", []) if audios: mm_data["audio"] = audios - # Text-only input not supported in composite processor - if not mm_data.get("audio", []): - prompt_ids = self.info.get_tokenizer().encode(prompt) - prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") - - feature_extractor = self.info.get_feature_extractor(**mm_kwargs) - mm_kwargs = dict( - **mm_kwargs, + hf_processor_mm_kwargs = dict( + **hf_processor_mm_kwargs, sampling_rate=feature_extractor.sampling_rate, ) - return super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - ) + return mm_data, hf_processor_mm_kwargs def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index 622376f0b3..a3691742ff 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -1195,79 +1195,95 @@ Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder class Qwen3OmniMoeThinkerMultiModalProcessor( Qwen2_5OmniThinkerMultiModalProcessor, ): - def _call_hf_processor( + def _get_hf_mm_data( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - mm_data = dict(mm_data) - audios = mm_data.pop("audios", []) + mm_items: MultiModalDataItems, + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + processor_data, passthrough_data = super()._get_hf_mm_data(mm_items) - def pad_to_hop_length(x: np.ndarray, hop_length: int) -> np.ndarray: - length = x.shape[-1] - if length % hop_length != 0: - pad_length = hop_length - (length % hop_length) - x = np.pad(x, (0, pad_length), mode="constant", constant_values=0) - return x - - # NOTE: WhisperFeatureExtractor cannot handle empty list of audios - feature_extractor = self.info.get_feature_extractor(**mm_kwargs) - hop_length = feature_extractor.hop_length + audios = processor_data.get("audios") if audios: - # NOTE: Qwen3-Omni processor accept "audio" + feature_extractor = self.info.get_feature_extractor() + hop_length = feature_extractor.hop_length + + def pad_to_hop_length(x: np.ndarray) -> np.ndarray: + length = x.shape[-1] + if length % hop_length != 0: + pad_length = hop_length - (length % hop_length) + x = np.pad(x, (0, pad_length), mode="constant", constant_values=0) + return x + # To make sure the cache works with padding=True, we pre-padded - # the audio to multiple of hop_length. - mm_data["audio"] = [ - pad_to_hop_length(audio, hop_length) + # the audio to a multiple of hop_length. + processor_data = dict(processor_data) + processor_data["audios"] = [ + pad_to_hop_length(audio) if isinstance(audio, np.ndarray) - else (pad_to_hop_length(audio[0], hop_length), audio[1]) + else (pad_to_hop_length(audio[0]), audio[1]) for audio in audios ] + return processor_data, passthrough_data + + def _apply_hf_processor_main( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> BatchFeature: + if mm_items.get_all_counts().get("audio", 0): # TODO(Isotr0py): Remove this patch after upstream fix PR # released and Transformers version update: # https://github.com/huggingface/transformers/pull/41473 - mm_kwargs = dict(mm_kwargs) - mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {}) - mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {}) - elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"): - # mm_data can be empty on a multimodal-processor-cache hit, where - # there's nothing to (re-)process this call and the real result - # comes from the cache — not a genuine "no audio" case. - raise ValueError( - "Video doesn't have audio track with `audio_in_video=True`" + hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs) + hf_processor_mm_kwargs["audio_kwargs"] = dict( + hf_processor_mm_kwargs.get("audio_kwargs") or {} + ) + hf_processor_mm_kwargs["text_kwargs"] = dict( + hf_processor_mm_kwargs.get("text_kwargs") or {} ) - hf_inputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, + processed_data = super()._apply_hf_processor_main( + mm_items, + hf_processor_mm_kwargs, ) if ( - "audio_feature_lengths" in hf_inputs - and "feature_attention_mask" in hf_inputs - and (audios := mm_data.get("audio", [])) + "audio_feature_lengths" in processed_data + and "feature_attention_mask" in processed_data ): - audio_num_frames = [] - for _, audio in enumerate(audios): - audio_length = len(audio[0]) if isinstance(audio, tuple) else len(audio) - num_frame = ( - (audio_length // hop_length) - if audio_length % hop_length == 0 - else (audio_length // hop_length - 1) + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + processor_data, _ = self._get_hf_mm_data(valid_mm_items) + audios = processor_data.get("audios", []) + if audios: + feature_extractor = self.info.get_feature_extractor( + **hf_processor_mm_kwargs ) - if mm_kwargs.get("truncation", False): - num_frame = min( - num_frame, feature_extractor.n_samples // hop_length + hop_length = feature_extractor.hop_length + + audio_num_frames = [] + for audio in audios: + audio_length = ( + len(audio[0]) if isinstance(audio, tuple) else len(audio) ) - audio_num_frames.append(num_frame) - hf_inputs["feature_attention_mask"] = [ - torch.ones(num_frame) for num_frame in audio_num_frames - ] - hf_inputs["audio_feature_lengths"] = torch.tensor(audio_num_frames) - return hf_inputs + num_frame = ( + (audio_length // hop_length) + if audio_length % hop_length == 0 + else (audio_length // hop_length - 1) + ) + if hf_processor_mm_kwargs.get("truncation", False): + num_frame = min( + num_frame, feature_extractor.n_samples // hop_length + ) + audio_num_frames.append(num_frame) + + processed_data["feature_attention_mask"] = [ + torch.ones(num_frame) for num_frame in audio_num_frames + ] + processed_data["audio_feature_lengths"] = torch.tensor(audio_num_frames) + + return processed_data def _maybe_apply_prompt_updates( self, diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index bc31091cfe..9390b7bacb 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -1280,12 +1280,21 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) proc_impl = getattr(type(hf_processor), "replace_video_token", None) return proc_impl is not None and proc_impl is not mixin_impl - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + mm_data = dict(mm_data) # Separate video processing from image processing. Because the videos @@ -1314,8 +1323,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # NOTE: a copy of is created to update do_sample_frames, # otherwise mm_hash for the object will be incorrect. - video_mm_kwargs = dict(**mm_kwargs) - merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs) + video_mm_kwargs = dict(**hf_processor_mm_kwargs) + merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs) if merged.keys() & {"size", "min_pixels", "max_pixels"}: video_size = dict(self.info.get_video_processor().size) size_override = merged.get("size") @@ -1365,10 +1374,13 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs: video_mm_kwargs["fps"] = None - video_outputs = super()._call_hf_processor( - prompt="<|vision_start|><|video_pad|><|vision_end|>", - mm_data=video_mm_data, - mm_kwargs=video_mm_kwargs, + video_outputs = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**video_mm_kwargs), + dict( + text="<|vision_start|><|video_pad|><|vision_end|>", + **video_mm_data, + ), + video_mm_kwargs, ) # Discard HF output input_ids — we use get_video_repl below @@ -1424,12 +1436,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # fps/num_frames are video-only kwargs already consumed by the loop; # exclude them so the text/image processor call below never gets a list. non_video_mm_kwargs = { - k: v for k, v in mm_kwargs.items() if k not in ("fps", "num_frames") + k: v + for k, v in hf_processor_mm_kwargs.items() + if k not in ("fps", "num_frames") } - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=non_video_mm_kwargs, + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**non_video_mm_kwargs), + dict(text=prompt_text, **mm_data), + non_video_mm_kwargs, ) # Replace each placeholder with pre-computed video tokens. @@ -1443,20 +1457,19 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) hf_config.video_token_id, hf_config.vision_end_token_id, ] - input_ids = processed_outputs.pop("input_ids") + input_ids = processed_data.pop("input_ids") if not isinstance(input_ids, list): input_ids = input_ids.tolist() (prompt_ids,) = input_ids expanded_ids = _replace_video_token_placeholders( prompt_ids, video_target, video_input_ids_lst ) - processed_outputs["input_ids"] = [expanded_ids] + processed_data["input_ids"] = [expanded_ids] - combined_outputs = dict( - processed_outputs, - **video_outputs, - ) - return BatchFeature(combined_outputs) + processed_data.update(video_outputs) + processed_data.update(passthrough_data) + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index f373a2cccb..255c67cf1a 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -345,28 +345,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo] return mm_fields - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> "BatchFeature": - """Run the HF processor and unpad inputs.""" - try: - hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs) - except ValueError: - if any(mm_data.values()): - raise - # Some processors reject a prompt holding placeholders with - # no data to go with them, so tokenize it without them - prompt_ids = self.info.get_tokenizer().encode(prompt) - hf_inputs = transformers.BatchFeature( - dict(input_ids=[prompt_ids]), tensor_type="pt" - ) - self._unpad_images(hf_inputs) - self._unpad_audios(hf_inputs, mm_data, mm_kwargs) - return hf_inputs - def _unpad_audios( self, hf_inputs: "BatchFeature", @@ -450,19 +428,6 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase): deriving them from updates.""" return [] - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> "BatchFeature": - """Ask which modality owns each token of the expanded prompt, which is the - only marking of the tokens an expansion adds around an item.""" - if any(mm_data.values()): - mm_data = {**mm_data, "return_mm_token_type_ids": True} - - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) - def _get_mm_token_ids(self, modality: str) -> list[int]: """Token ids marking where `modality` sits in the prompt, which for some processors differ from the placeholder written into it. @@ -612,24 +577,55 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase): with timing_ctx.record("apply_hf_processor"): hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - prompt = hf_processor.decode(prompt) + prompt_text = hf_processor.decode(prompt) # Bypass cached processor and always apply to the full set of mm inputs # NOTE: we can't just set caching=False because base class method # transforms outputs to `MultiModalKwargs` which is not going to # work for Transformers. The vision path has logic tied to # `mm_tokens_per_modality` in _apply_vision() - prompt_ids, processed_data = self._apply_hf_processor_text_mm( - prompt_text=prompt, - mm_items=mm_items, - hf_processor_mm_kwargs={ - **hf_processor_mm_kwargs, - # HF processors only accept text, and the decoded string already - # contains any special tokens, so don't let them be added again - # (`_call_hf_processor` disables `add_special_tokens`). - "add_special_tokens": False, - }, + hf_processor_mm_kwargs = { + **hf_processor_mm_kwargs, + # HF processors only accept text, and the decoded string already + # contains any special tokens, so don't let them be added again + # (the HF processor call disables `add_special_tokens`). + "add_special_tokens": False, + } + + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} ) + processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + # Ask which modality owns each token of the expanded prompt, which + # is the only marking of the tokens an expansion adds around an item + if any(processor_data.values()): + processor_data = {**processor_data, "return_mm_token_type_ids": True} + + try: + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=prompt_text, **processor_data), + hf_processor_mm_kwargs, + ) + except ValueError: + if any(processor_data.values()): + raise + # Some processors reject a prompt holding placeholders with + # no data to go with them, so tokenize it without them + prompt_ids = self.info.get_tokenizer().encode(prompt_text) + processed_data = transformers.BatchFeature( + dict(input_ids=[prompt_ids]), tensor_type="pt" + ) + self._unpad_images(processed_data) + self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs) + processed_data.update(passthrough_data) + + input_ids = processed_data.pop("input_ids") + if not isinstance(input_ids, list): + input_ids = input_ids.tolist() + + (prompt_ids,) = input_ids # Use overrides if provided; fallback to data-dependent hashing. with timing_ctx.record("get_mm_hashes"): @@ -773,17 +769,41 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): except (AttributeError, KeyError, TypeError): return None - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> "BatchFeature": - """Ask for the replacement each placeholder expands to, and record it as - per-item fields: its token ids, and the tokens or patches behind them.""" + mm_counts = mm_items.get_all_counts() + + valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0}) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_counts) + + # Ask for the replacement each placeholder expands to, and record it as + # per-item fields: its token ids, and the tokens or patches behind them if has_mm_data := any(mm_data.values()): mm_data = {**mm_data, "return_text_replacement_offsets": True} - hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs) + + try: + hf_inputs = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + dict(text=prompt_text, **mm_data), + hf_processor_mm_kwargs, + ) + except ValueError: + if any(mm_data.values()): + raise + # Some processors reject a prompt holding placeholders with + # no data to go with them, so tokenize it without them + tokenizer = self.info.get_tokenizer() + hf_inputs = transformers.BatchFeature( + dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt" + ) + self._unpad_images(hf_inputs) + self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs) + # Drop the inputs the model would reject hf_inputs.pop("mm_token_type_ids", None) hf_inputs.pop("token_type_ids", None) @@ -802,6 +822,8 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): "so it can be fixed, and install `transformers<5.15.0` in the " "meantime to locate placeholders in the expanded prompt instead." ) + hf_inputs.update(passthrough_data) + hf_inputs.pop("input_ids", None) return hf_inputs tokenizer = self.info.get_tokenizer() @@ -829,6 +851,9 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): counts.append(int(ids.eq(_get_embed_token_id(ids)).sum())) hf_inputs["num_audio_tokens"] = torch.tensor(counts) + hf_inputs.update(passthrough_data) + hf_inputs.pop("input_ids") + return hf_inputs diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index 81e97ce4ce..60dc1fb133 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -190,41 +190,84 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]) class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - # Text-only input not supported in composite processor - if not mm_data.get("audios", []): - prompt_ids = self.info.get_tokenizer().encode( - prompt, add_special_tokens=False - ) - prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) - mm_data = dict(mm_data) - audios = mm_data.pop("audios", []) + def _preprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + audios = mm_data.get("audios", []) assert isinstance(audios, list) - feature_extractor = self.info.get_feature_extractor(**mm_kwargs) - mm_kwargs = dict( - **mm_kwargs, + feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + hf_processor_mm_kwargs = dict( + **hf_processor_mm_kwargs, sampling_rate=feature_extractor.sampling_rate, include_audio_num_chunks=True, ) - item_processor_data = dict(**mm_data, audios=audios) + return mm_data, hf_processor_mm_kwargs - output = super()._call_hf_processor( - prompt=prompt, - mm_data=item_processor_data, - mm_kwargs=mm_kwargs, + def _apply_hf_processor_main( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} ) - output["audio_features"] = output.pop("audio_values") + processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - return output + if processor_data: + processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data( + processor_data, + hf_processor_mm_kwargs, + ) + + prompt_text = self._get_hf_processor_text(mm_items.get_all_counts()) + if prompt_text is not None: + processor_data = dict(text=prompt_text, **processor_data) + + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + + def patched_call(**kwargs) -> BatchFeature: + # The remote UltravoxProcessor does not support the + # `truncation` kwarg injected by `call_hf_processor`: it + # passes `truncation` to its audio processor by itself while + # also forwarding `**kwargs` there, causing a duplicate + # keyword argument error. Its hardcoded `truncation=False` + # already matches vLLM's intended behavior. + kwargs.pop("truncation", None) + + return hf_processor(**kwargs) + + processed_data = self.info.ctx.call_hf_processor( + patched_call, + processor_data, + hf_processor_mm_kwargs, + ) + processed_data.update(passthrough_data) + else: + processed_data = BatchFeature(dict(passthrough_data)) + + return self._postprocess_hf_mm_data( + processor_data, + hf_processor_mm_kwargs, + processed_data, + ) + + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if "audio_values" in processed_data: + processed_data["audio_features"] = processed_data.pop("audio_values") + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/unlimited_ocr.py b/vllm/model_executor/models/unlimited_ocr.py index 0f29337314..c96e5f293d 100644 --- a/vllm/model_executor/models/unlimited_ocr.py +++ b/vllm/model_executor/models/unlimited_ocr.py @@ -225,7 +225,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor): self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> tuple[list[int], MultiModalProcessingInfo, bool]: + ) -> MultiModalProcessingInfo: # The processor logic differs for single-image (crop) vs multi-image # (no crop) requests. The processing cache assumes per-item output is # invariant of how many images are passed per prompt, so we only cache diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index 2013d48ce5..07a7f9c325 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -47,12 +47,10 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, - MultiModalProcessingInfo, PlaceholderFeaturesInfo, ProcessorInputs, PromptReplacement, PromptUpdate, - TimingContext, ) from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config @@ -204,6 +202,9 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]): class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]): + # The tokens are already inserted by the chat template + hf_processor_applies_updates = True + def _get_mm_fields_config( self, hf_inputs: Mapping[str, NestedTensors], @@ -220,12 +221,14 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]) # skip validation here pass - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _preprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: mm_data = dict(mm_data) audios = mm_data.pop("audios", []) @@ -233,18 +236,23 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]) # MistralCommonVoxtralProcessor accepts "audio" mm_data["audio"] = audios - outputs = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**mm_kwargs), - dict(text=prompt, **mm_data), - # Avoid padding issue - dict(**mm_kwargs, return_tensors=None), - ) + # Avoid padding issue + hf_processor_mm_kwargs = dict(**hf_processor_mm_kwargs, return_tensors=None) + return mm_data, hf_processor_mm_kwargs + + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: # Missing batch dimension - if is_list_of(outputs["input_ids"], int): - outputs["input_ids"] = [outputs["input_ids"]] + input_ids = processed_data.get("input_ids") + if is_list_of(input_ids, int): + processed_data["input_ids"] = [input_ids] - return outputs + return processed_data def _get_prompt_updates( self, @@ -287,16 +295,6 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]) ), ] - def _cached_apply_hf_processor( - self, - inputs: ProcessorInputs, - timing_ctx: TimingContext, - ) -> tuple[list[int], MultiModalProcessingInfo, bool]: - prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx) - - # NOTE: The tokens are already inserted by the chat template - return prompt_ids, mm_info, True - @MULTIMODAL_REGISTRY.register_processor( VoxtralMultiModalProcessor, diff --git a/vllm/model_executor/models/whisper.py b/vllm/model_executor/models/whisper.py index 0486933c91..d21903bd65 100644 --- a/vllm/model_executor/models/whisper.py +++ b/vllm/model_executor/models/whisper.py @@ -756,27 +756,36 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo # for encoder profiling. return [0] - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) + + def _preprocess_hf_mm_data( self, - prompt: str, mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - if mm_data: - feature_extractor = self.info.get_feature_extractor(**mm_kwargs) - mm_data = dict(audio=mm_data.pop("audios")) - mm_kwargs = dict( - **mm_kwargs, - sampling_rate=feature_extractor.sampling_rate, - ) - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs) + + mm_data = dict(mm_data) + mm_data["audio"] = mm_data.pop("audios") + + hf_processor_mm_kwargs = dict( + **hf_processor_mm_kwargs, + sampling_rate=feature_extractor.sampling_rate, ) - if "labels" in processed_outputs: - processed_outputs["input_ids"] = processed_outputs.pop("labels") - return processed_outputs + + return mm_data, hf_processor_mm_kwargs + + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, + ) -> BatchFeature: + if "labels" in processed_data: + processed_data["input_ids"] = processed_data.pop("labels") + + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/models/dots3_note/common/processor.py b/vllm/models/dots3_note/common/processor.py index 25c8c5de58..724d9b5ac1 100644 --- a/vllm/models/dots3_note/common/processor.py +++ b/vllm/models/dots3_note/common/processor.py @@ -649,13 +649,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn processor_data["videos"] = raw_videos return processor_data, passthrough_data - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + return self.dummy_inputs.get_dummy_text(mm_counts) def _get_mm_fields_config( self, diff --git a/vllm/models/inkling/common/mm_preprocess.py b/vllm/models/inkling/common/mm_preprocess.py index df1464e319..249d7edeb4 100644 --- a/vllm/models/inkling/common/mm_preprocess.py +++ b/vllm/models/inkling/common/mm_preprocess.py @@ -22,7 +22,7 @@ from vllm.multimodal.inputs import ( MultiModalFieldConfig, MultiModalKwargsItems, ) -from vllm.multimodal.parse import MultiModalDataParser +from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser from vllm.multimodal.processing import ( BaseDummyInputsBuilder, BaseMultiModalProcessor, @@ -135,8 +135,8 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]): num_images = mm_counts.get("image", 0) num_audios = mm_counts.get("audio", 0) # Use spellings the renderer would emit; tokenization is bypassed in - # _call_hf_processor (we build input_ids directly), so the exact text - # only needs to be a stable per-item marker. + # _apply_hf_processor_main (we build input_ids directly), so the exact + # text only needs to be a stable per-item marker. return ("<|content_image|>" * num_images) + ( "<|content_audio_input|>" * num_audios ) @@ -179,17 +179,19 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]): class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: - # Inkling is not a standard HF processor (no fused text+mm call), so we run - # the vendored extractors ourselves and tokenize the text separately. - # The MM placeholders in `prompt` are expanded later by the prompt - # updates, so here we emit ONE placeholder id per media item. - processor = self.info.get_hf_processor(**mm_kwargs) + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) tokenizer = self.info.get_tokenizer() images = mm_data.get("images") or [] @@ -200,7 +202,7 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]) audios = list(cast(Iterable[Any], audios)) prompt_ids = self._tokenize_with_placeholders( - prompt, tokenizer, len(images), len(audios) + prompt_text, tokenizer, len(images), len(audios) ) data: dict[str, Any] = {"input_ids": [prompt_ids]} @@ -232,7 +234,9 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]) data["input_audio_features"] = input_audio_features data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64) - return BatchFeature(data=data, tensor_type=None) + processed_data = BatchFeature(data=data, tensor_type=None) + processed_data.update(passthrough_data) + return processed_data def _tokenize_with_placeholders( self, diff --git a/vllm/models/kimi_k3/common/mm_preprocess.py b/vllm/models/kimi_k3/common/mm_preprocess.py index bc8386b4a6..70c017fd72 100644 --- a/vllm/models/kimi_k3/common/mm_preprocess.py +++ b/vllm/models/kimi_k3/common/mm_preprocess.py @@ -231,19 +231,6 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]): class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]): """Image-only multi-modal processor for Kimi-K3.""" - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - # Override so the base always routes through the text+mm path - # (`KimiK3Processor.__call__`). Otherwise the mm-only fast path calls - # the checkpoint image processor directly with bare PIL images, but it - # requires `{"type": "image", "image": PIL}` media dicts that only our - # wrapper builds. - return super()._call_hf_processor(prompt, mm_data, mm_kwargs) - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/models/minimax_m3/common/mm_preprocess.py b/vllm/models/minimax_m3/common/mm_preprocess.py index 1481da4dc8..86b702585f 100644 --- a/vllm/models/minimax_m3/common/mm_preprocess.py +++ b/vllm/models/minimax_m3/common/mm_preprocess.py @@ -306,12 +306,21 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing class MiniMaxM3VLMultiModalProcessor( BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo] ): - def _call_hf_processor( + def _apply_hf_processor_main( self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], ) -> BatchFeature: + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + + if not mm_data: + return BatchFeature(dict(passthrough_data)) + + prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) + mm_data = dict(mm_data) # With ``video_needs_metadata=True`` each video arrives as a # ``(frames, metadata)`` tuple. Split the frames back out and forward the @@ -340,15 +349,17 @@ class MiniMaxM3VLMultiModalProcessor( # Override the video processor's default do_resize=False (set for a # pre-resized pipeline) to True for vLLM's raw-frame inputs. - merged = dict(do_resize=True, **mm_kwargs) - data = dict(text=prompt, **mm_data) + merged = dict(do_resize=True, **hf_processor_mm_kwargs) + data = dict(text=prompt_text, **mm_data) if video_metadata is not None: data["video_metadata"] = video_metadata - return self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**mm_kwargs), + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), data, merged, ) + processed_data.update(passthrough_data) + return processed_data def _get_mm_fields_config( self, diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py index 45acdf8f4d..f13ca6ab88 100644 --- a/vllm/multimodal/processing/context.py +++ b/vllm/multimodal/processing/context.py @@ -262,9 +262,6 @@ class InputProcessingContext: hf_processor: Callable[..., BatchFeature] | ProcessorMixin, data: Mapping[str, object], kwargs: Mapping[str, object] = {}, - *, - num_tries: int = 1, - max_tries: int = 5, ) -> BatchFeature: """ Call `hf_processor` on the prompt `data` @@ -274,6 +271,12 @@ class InputProcessingContext: merged_kwargs = self.get_merged_mm_kwargs(kwargs) + # vLLM needs the full untruncated sequence to keep multi-modal + # placeholder tokens aligned; note that the text inputs in + # call_hf_processor are just dummy text, not the original prompt. + # The original prompt is already tokenized by the renderer. + merged_kwargs.setdefault("truncation", False) + allowed_kwargs = get_allowed_kwarg_only_overrides( hf_processor, merged_kwargs, diff --git a/vllm/multimodal/processing/dummy_inputs.py b/vllm/multimodal/processing/dummy_inputs.py index 09df2e0532..ac3fe54721 100644 --- a/vllm/multimodal/processing/dummy_inputs.py +++ b/vllm/multimodal/processing/dummy_inputs.py @@ -91,10 +91,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]): # prompt tokens. dummy_prompt = [] else: - dummy_prompt = tokenizer.encode( - dummy_text, - **self.info.default_tok_params.get_encode_kwargs(), - ) + dummy_prompt = tokenizer.encode(dummy_text, truncation=False) return ProcessorInputs( prompt=dummy_prompt, diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index 599db7406a..17bcc2f6dd 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -5,9 +5,10 @@ from collections import defaultdict, deque from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence from dataclasses import dataclass, field, replace from enum import Enum -from functools import lru_cache, partial +from functools import lru_cache from typing import ( TYPE_CHECKING, + ClassVar, Generic, NamedTuple, Protocol, @@ -28,7 +29,6 @@ from vllm.inputs import ( ) from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike -from vllm.transformers_utils.processor import call_hf_processor_mm_only from vllm.utils.collection_utils import flatten_2d_lists, full_groupby from ..inputs import ( @@ -1177,6 +1177,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): Not to be confused with `transformers.ProcessorMixin`. """ + hf_processor_applies_updates: ClassVar[bool] = False + """ + Only set to True if the tokenizer or chat template + (which are run before MM processing) inserts placeholder tokens. + """ + def __init__( self, info: _I, @@ -1302,131 +1308,100 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): return processor_data, passthrough_data - def _call_hf_processor( + def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None: + """ + Get the text to pass to the HF processor alongside the multi-modal + data. + + By default, no text is passed. If the HF processor requires that + text and multi-modal items correspond to each other, you should + override this method to return dummy text generated by + [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]. + """ + return None + + def _preprocess_hf_mm_data( self, - prompt: str, - # Not to be confused with `mm_data` in `self.apply`. - # This refers to the data to be passed to HF processor. mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + """ + Pre-process the multi-modal data and HF processor keyword arguments + before they are passed to the HF processor. + + By default, both are returned as-is. If the HF processor expects the + multi-modal data under different keys than those provided by the + multi-modal items (e.g. `audio` instead of `audios`), or requires + additional keyword arguments (e.g. `sampling_rate`), you should + override this method. + """ + return mm_data, hf_processor_mm_kwargs + + def _postprocess_hf_mm_data( + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], + processed_data: BatchFeature, ) -> BatchFeature: """ - Call the HF processor on the prompt text and - associated multi-modal data. + Post-process the output of the HF processor. + + By default, the output is returned as-is. If you need to modify the + output of the HF processor before it is converted into multi-modal + keyword arguments, you should override this method. """ - return self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**mm_kwargs), - dict(text=prompt, **mm_data), - mm_kwargs, - ) - - def _apply_hf_processor_text_mm( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[list[int], BatchFeature]: - """ - Apply the HF processor on the prompt text and multi-modal data - together. - """ - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - processed_data = self._call_hf_processor( - prompt=prompt_text, - mm_data=processor_data, - mm_kwargs=hf_processor_mm_kwargs, - ) - processed_data.update(passthrough_data) - - input_ids = processed_data.pop("input_ids") - if not isinstance(input_ids, list): - input_ids = input_ids.tolist() - - (prompt_ids,) = input_ids - - return prompt_ids, processed_data - - def _apply_hf_processor_tokens_only( - self, - prompt_tokens: list[int], - ) -> list[int]: - """ - Apply the HF processor on the prompt tokens only. - - Most HF processors accept prompt text but not prompt tokens. - If the HF processor adds or removes tokens that are not related to - multi-modal data, you should override this method so it is consistent - with the tokenization of the corresponding text. - """ - return prompt_tokens - - def _apply_hf_processor_mm_only( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: - """ - Apply the HF processor on the multi-modal data only. - - Since HF processor requires that text and multi-modal items - correspond to each other, we generate dummy text using - [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder] - to go along with the multi-modal data. - """ - # Custom logic based on text inputs - if type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor: - mm_counts = mm_items.get_all_counts() - - _, mm_processed_data = self._apply_hf_processor_text_mm( - prompt_text=self.dummy_inputs.get_dummy_text(mm_counts), - mm_items=mm_items, - hf_processor_mm_kwargs=hf_processor_mm_kwargs, - ) - - return mm_processed_data - - valid_mm_items = mm_items.select( - {k for k, c in mm_items.get_all_counts().items() if c > 0} - ) - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - - processed_data = self.info.ctx.call_hf_processor( - partial( - call_hf_processor_mm_only, - self.info.get_hf_processor(**hf_processor_mm_kwargs), - ), - processor_data, - hf_processor_mm_kwargs, - ) - processed_data.update(passthrough_data) - return processed_data def _apply_hf_processor_main( self, - prompt: list[int], mm_items: MultiModalDataItems, hf_processor_mm_kwargs: Mapping[str, object], - ) -> tuple[list[int], BatchFeature, bool]: + ) -> BatchFeature: """ - Apply the HF processor on the prompt tokens and multi-modal data. - - In addition, return whether prompt updates have been applied - (`False` here, since for token prompts the updates are normally - applied separately via `_maybe_apply_prompt_updates`). + Apply the HF processor on the multi-modal data. """ - prompt_ids = self._apply_hf_processor_tokens_only(prompt) + valid_mm_items = mm_items.select( + {k for k, c in mm_items.get_all_counts().items() if c > 0} + ) + processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) - mm_processed_data = self._apply_hf_processor_mm_only( - mm_items=mm_items, - hf_processor_mm_kwargs=hf_processor_mm_kwargs, + if processor_data: + processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data( + processor_data, hf_processor_mm_kwargs + ) + + prompt_text = self._get_hf_processor_text(mm_items.get_all_counts()) + if prompt_text is not None: + processor_data = dict(text=prompt_text, **processor_data) + + processed_data = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_processor_mm_kwargs), + processor_data, + hf_processor_mm_kwargs, + ) + processed_data.update(passthrough_data) + else: + from transformers.feature_extraction_utils import BatchFeature + + processed_data = BatchFeature(dict(passthrough_data)) + + return self._postprocess_hf_mm_data( + processor_data, + hf_processor_mm_kwargs, + processed_data, ) - return prompt_ids, mm_processed_data, False + def _postprocess_prompt(self, prompt: list[int]) -> list[int]: + """ + Post-process the prompt token IDs before locating or applying + multi-modal prompt updates. + + By default, the prompt is returned as-is. If the HF processor (or + chat template) applies additional transformations to the prompt + that are not reflected in its multi-modal outputs, you should + override this method to replicate them. + """ + return prompt def _get_cache_missing_items( self, @@ -1531,14 +1506,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> tuple[list[int], MultiModalProcessingInfo, bool]: + ) -> MultiModalProcessingInfo: with timing_ctx.record("apply_hf_processor"): - ( - prompt_ids, - mm_processed_data, - is_update_applied, - ) = self._apply_hf_processor_main( - prompt=inputs.prompt, + mm_processed_data = self._apply_hf_processor_main( mm_items=inputs.mm_data_items, hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs, ) @@ -1569,13 +1539,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): prompt_updates=mm_prompt_updates, ) - return prompt_ids, mm_info, is_update_applied + return mm_info def _cached_apply_hf_processor( self, inputs: ProcessorInputs, timing_ctx: TimingContext, - ) -> tuple[list[int], MultiModalProcessingInfo, bool]: + ) -> MultiModalProcessingInfo: """ Apply the HF processor on the full prompt text, caching the results and reusing cached results. @@ -1599,16 +1569,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): mm_hashes=mm_hashes, ) - # NOTE: `prompt` does not correspond to `mm_missing_data_items`, + # NOTE: The prompt does not correspond to `mm_missing_data_items`, # so we can't apply prompt updates until the new multimodal # items are combined with the cached multimodal items with timing_ctx.record("apply_hf_processor"): - ( - prompt_ids, - mm_missing_processed_data, - is_update_applied, - ) = self._apply_hf_processor_main( - prompt=inputs.prompt, + mm_missing_processed_data = self._apply_hf_processor_main( mm_items=mm_missing_data_items, hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs, ) @@ -1641,7 +1606,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): prompt_updates=mm_prompt_updates, ) - return prompt_ids, mm_info, is_update_applied + return mm_info def _apply_token_matches( self, @@ -1770,7 +1735,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): "There is likely a problem with your " "implementation of merged multi-modal processor for this " "model (usually arising from an inconsistency between " - "`_call_hf_processor` and `_get_mm_fields_config`)." + "`_apply_hf_processor_main` and `_get_mm_fields_config`)." ) def _validate_mm_updates( @@ -1805,8 +1770,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): f"Expected there to be {item_count} prompt placeholders " f"corresponding to {item_count} {modality} items, but " f"instead found {len(placeholders)} prompt placeholders! " - "Make sure the implementation of `_call_hf_processor` and " - "`_get_mm_fields_config` are consistent with each other." + "Make sure the implementation of `_apply_hf_processor_main` " + "and `_get_mm_fields_config` are consistent with each other." ) def _maybe_apply_prompt_updates( @@ -1854,11 +1819,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): 3. Extract information about the placeholder tokens from the processed token IDs. """ - ( - prompt_ids, - mm_info, - is_update_applied, - ) = self._cached_apply_hf_processor(inputs, timing_ctx) + prompt_ids = self._postprocess_prompt(inputs.prompt) + mm_info = self._cached_apply_hf_processor(inputs, timing_ctx) with timing_ctx.record("apply_prompt_updates"): prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates( @@ -1866,7 +1828,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): prompt_ids=prompt_ids, mm_kwargs=mm_info.kwargs, mm_prompt_updates=mm_info.prompt_updates, - is_update_applied=is_update_applied, + is_update_applied=self.hf_processor_applies_updates, ) mm_placeholder_ranges = { diff --git a/vllm/transformers_utils/processor.py b/vllm/transformers_utils/processor.py index ee73393f6b..64a94950d4 100644 --- a/vllm/transformers_utils/processor.py +++ b/vllm/transformers_utils/processor.py @@ -11,17 +11,13 @@ from transformers import ( AutoImageProcessor, AutoProcessor, AutoVideoProcessor, - BatchFeature, processing_utils, ) -from transformers.audio_utils import AudioInput from transformers.feature_extraction_utils import FeatureExtractionMixin from transformers.image_processing_utils import BaseImageProcessor -from transformers.image_utils import ImageInput from transformers.models.auto.video_processing_auto import VIDEO_PROCESSOR_MAPPING_NAMES from transformers.processing_utils import ProcessorMixin from transformers.video_processing_utils import BaseVideoProcessor -from transformers.video_utils import VideoInput from typing_extensions import TypeVar from vllm.logger import init_logger @@ -554,43 +550,3 @@ def cached_video_processor_from_config( processor_cls_overrides=processor_cls, # type: ignore[arg-type] **_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs), ) - - -def call_hf_processor_mm_only( - processor: ProcessorMixin, - images: ImageInput | None = None, - videos: VideoInput | None = None, - audio: AudioInput | None = None, - **kwargs, -) -> BatchFeature: - output_kwargs = processor._merge_kwargs( - get_processor_kwargs_type(processor), - **kwargs, - ) - - if audio is not None and ( - feature_extractor := getattr(processor, "feature_extractor", None) - ): - audio_inputs = feature_extractor(audio, **output_kwargs["audio_kwargs"]) - audio_inputs["feature_attention_mask"] = audio_inputs.pop("attention_mask") - else: - audio_inputs = {} - - if images is not None and ( - image_processor := getattr(processor, "image_processor", None) - ): - images_inputs = image_processor(images=images, **output_kwargs["images_kwargs"]) - else: - images_inputs = {} - - if videos is not None and ( - video_processor := getattr(processor, "video_processor", None) - ): - videos_inputs = video_processor(videos=videos, **output_kwargs["videos_kwargs"]) - else: - videos_inputs = {} - - return BatchFeature( - data={**audio_inputs, **images_inputs, **videos_inputs}, - tensor_type=kwargs.get("return_tensors"), - )