[MM] Simplify _apply_hf_processor_main (#53275)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-22 01:49:37 -07:00
committed by GitHub
parent 7ca49fbe4b
commit 0b19ebcacd
90 changed files with 1848 additions and 1706 deletions
+49 -20
View File
@@ -153,11 +153,13 @@ def get_supported_mm_limits(self) -> Mapping[str, int | None]:
## 3. Specify dummy inputs ## 3. Specify dummy inputs
Then, inherit [BaseDummyInputsBuilder][vllm.multimodal.processing.BaseDummyInputsBuilder] to construct dummy inputs for Then, inherit [BaseDummyInputsBuilder][vllm.multimodal.processing.BaseDummyInputsBuilder] to construct the dummy inputs
HF processing. The processed outputs are also used for memory profiling. that are used for memory profiling.
Override the abstract methods [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text] and [get_dummy_mm_data][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_mm_data] to construct dummy inputs. These dummy inputs should result in the worst-case memory usage of the model so that vLLM can reserve the correct amount of memory for it. Override the abstract methods [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text] and [get_dummy_mm_data][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_mm_data] to construct dummy inputs. These dummy inputs should result in the worst-case memory usage of the model so that vLLM can reserve the correct amount of memory for it.
Besides profiling, the dummy text is passed to the HF processor by models whose HF processor requires text corresponding to the multi-modal items (see [Multi-modal fields](#multi-modal-fields)). If the HF processor can process multi-modal data without any text, the dummy text is not used during HF processing.
Assuming that the memory usage increases with the number of tokens, the dummy inputs can be constructed to maximize the number of output embeddings, which is the same number as placeholder feature tokens. Assuming that the memory usage increases with the number of tokens, the dummy inputs can be constructed to maximize the number of output embeddings, which is the same number as placeholder feature tokens.
=== "Basic example: LLaVA" === "Basic example: LLaVA"
@@ -446,37 +448,64 @@ return a schema of the tensors outputted by the HF processor that are related to
like in LLaVA, each image's features must be independent of the others (which like in LLaVA, each image's features must be independent of the others (which
is also required for prefix caching to work correctly). So, we un-pad each image is also required for prefix caching to work correctly). So, we un-pad each image
back to its own size by overriding back to its own size by overriding
[BaseMultiModalProcessor._call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor]: [BaseMultiModalProcessor._postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data]
to post-process the outputs of the HF processor:
??? code ??? code
```python ```python
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, # Mistral3Processor requires text corresponding to the images
prompt: str, return self.dummy_inputs.get_dummy_text(mm_counts)
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
pixel_values = processed_outputs.get("pixel_values") def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
pixel_values = processed_data.get("pixel_values")
if pixel_values is not None: if pixel_values is not None:
# Avoid padding since we need the output for each image to be # Avoid padding since we need the output for each image to be
# independent of other images for the cache to work correctly # independent of other images for the cache to work correctly
image_sizes = processed_outputs["image_sizes"] image_sizes = processed_data["image_sizes"]
assert len(pixel_values) == len(image_sizes) assert len(pixel_values) == len(image_sizes)
processed_outputs["pixel_values"] = [ processed_data["pixel_values"] = [
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes) p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
] ]
return processed_outputs return processed_data
``` ```
The default implementation of
[_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]
calls the HF processor on the multi-modal data without passing any text.
If the HF processor instead requires text corresponding to the multi-modal items,
you should override
[_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text]
to return the dummy text from
[BaseDummyInputsBuilder.get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text]
like in the example above. If the HF processor expects the multi-modal data
under different keys than those provided by the multi-modal items
(e.g. `audio` instead of `audios`), or requires additional keyword arguments
(e.g. `sampling_rate`), you should override
[_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data].
If you need to modify the output of the HF processor,
you should override
[_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data].
If the HF processor applies additional transformations to the prompt tokens
that are not reflected in its multi-modal outputs (e.g. prepending a BOS token),
you should override
[_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt]
to replicate them.
For even more control over how the HF processor is called, you can override
[_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]
directly.
Since `pixel_values` is now a list with one tensor per image, we can override Since `pixel_values` is now a list with one tensor per image, we can override
[_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows: [_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows:
@@ -680,7 +709,7 @@ Examples:
### Handling prompt updates unrelated to multi-modal data ### Handling prompt updates unrelated to multi-modal data
[_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates] assumes that each application of prompt update corresponds to one multi-modal item. If the HF processor performs additional processing regardless of how many multi-modal items there are, you should override [_apply_hf_processor_tokens_only][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_tokens_only] so that the processed token inputs are consistent with the result of applying the HF processor on text inputs. This is because token inputs bypass the HF processor according to [our design](../../design/mm_processing.md). [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates] assumes that each application of prompt update corresponds to one multi-modal item. If the HF processor performs additional processing on the prompt regardless of how many multi-modal items there are, you should override [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt] so that the prompt token IDs are consistent with the result of applying the HF processor on text inputs. This is because token inputs bypass the HF processor according to [our design](../../design/mm_processing.md).
Examples: Examples:
@@ -690,7 +719,7 @@ Examples:
### Custom HF processor ### Custom HF processor
Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and pass it to [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor]. Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. It is then applied to the multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor] inside [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main].
Examples: Examples:
+11 -42
View File
@@ -2,7 +2,15 @@
To enable various optimizations in vLLM such as [chunked prefill](../configuration/optimization.md#chunked-prefill) and [prefix caching](../features/automatic_prefix_caching.md), we use [BaseMultiModalProcessor][vllm.multimodal.processing.BaseMultiModalProcessor] to provide the correspondence between placeholder feature tokens (e.g. `<image>`) and multi-modal inputs (e.g. the raw input image) based on the outputs of HF processor. To enable various optimizations in vLLM such as [chunked prefill](../configuration/optimization.md#chunked-prefill) and [prefix caching](../features/automatic_prefix_caching.md), we use [BaseMultiModalProcessor][vllm.multimodal.processing.BaseMultiModalProcessor] to provide the correspondence between placeholder feature tokens (e.g. `<image>`) and multi-modal inputs (e.g. the raw input image) based on the outputs of HF processor.
Here are the main features of [BaseMultiModalProcessor][vllm.multimodal.processing.BaseMultiModalProcessor]: In vLLM's rendering pipeline (see [BaseRenderer][vllm.renderers.base.BaseRenderer]), tokenization is performed as a separate step before multi-modal processing. Therefore, `BaseMultiModalProcessor` needs to recreate the output of calling HF processor end-to-end, while not being able to see the original text. This is achieved through **Dummy Input Text** and **Prompt Update Detection**.
## Dummy Input Text
Since Transformers 5.10, `ProcessorMixin` now allows multi-modal inputs to be passed by themselves. However, certain subclasses (such as `ChameleonProcessor`) and older out-of-tree implementations may still define their own `__call__` method that assumes the presence of text with corresponding placeholder tokens. This causes a problem as we don't have the original text anymore to pass to these processors.
To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text] returns so that [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] passes it to the HF processor together with the multi-modal inputs to obtain the processed multi-modal data.
Similarly, since the multi-modal data extracted by vLLM may not match what a specific HF processor expects, [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] allows each model to adapt the inputs via [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data] (e.g. renaming keys like `audios` to `audio` or injecting extra keyword arguments such as `sampling_rate`) and the outputs via [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data], without having to reimplement the entire method.
## Prompt Update Detection ## Prompt Update Detection
@@ -13,46 +21,9 @@ One of the main responsibilities of HF processor is to update the prompt with pl
The information about which tokens have been updated is key to finding the correspondence between placeholder feature tokens and multi-modal inputs. The information about which tokens have been updated is key to finding the correspondence between placeholder feature tokens and multi-modal inputs.
In vLLM, this information is specified using [PromptUpdate][vllm.multimodal.processing.PromptUpdate] in [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates]. We can automatically detect whether HF has updated the prompt by checking the existence of the updated tokens. Since we call HF processor without the input text, we have to perform this update by ourselves. In vLLM, we represent the necessary information using [PromptUpdate][vllm.multimodal.processing.PromptUpdate] in [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates], and apply them via [_apply_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._apply_prompt_updates].
## Tokenized Prompt Inputs Some HF processors additionally transform the prompt itself regardless of the multi-modal inputs (such as `ChameleonProcessor` appending a sep token for chat mode). Since the prompt tokens likewise bypass the HF processor, such transformations are replicated via [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt] before the prompt updates are located or applied.
To enable tokenization in a separate process, we support passing input token IDs alongside multi-modal data.
### The problem
Consider that HF processors follow these main steps:
1. Tokenize the text
2. Process multi-modal inputs
3. Perform prompt updates
And we require that:
- For text + multi-modal inputs, apply all steps 1--3.
- For tokenized + multi-modal inputs, apply only steps 2--3.
How can we achieve this without rewriting HF processors? We can try to call the HF processor several times on different inputs:
- For text + multi-modal inputs, simply call the HF processor directly.
- For tokenized + multi-modal inputs, call the processor only on the multi-modal inputs.
While HF processors support text + multi-modal inputs natively, this is not so for tokenized + multi-modal inputs: an error is thrown if the number of input placeholder tokens do not correspond to the number of multi-modal inputs.
Moreover, since the tokenized text has not passed through the HF processor, we have to apply Step 3 by ourselves to keep the output tokens and multi-modal data consistent with each other.
### Dummy text
We work around the first issue by requiring each model to define how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text]. This lets us generate dummy text corresponding to the multi-modal inputs and input them together to obtain the processed multi-modal data.
### Automatic prompt updating
We address the second issue by implementing model-agnostic code in
[_apply_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._apply_prompt_updates] to automatically update the prompt with feature placeholder tokens based on the specification outputted by [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates].
### Summary
With the help of dummy text and automatic prompt updating, our multi-modal processor can finally accept both text and token prompts with multi-modal data. The detailed logic is shown in [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main].
## Processor Output Caching ## Processor Output Caching
@@ -60,8 +31,6 @@ Some HF processors, such as the one for Qwen2-VL, are [very slow](https://github
When new data is passed in, we first check which items are in the cache, and which ones are missing. The missing items are passed into the HF processor in a single batch and cached, before being merged with the existing items in the cache. When new data is passed in, we first check which items are in the cache, and which ones are missing. The missing items are passed into the HF processor in a single batch and cached, before being merged with the existing items in the cache.
Since we only process the missing multi-modal data items, the number of input placeholder tokens no longer corresponds to the number of the multi-modal inputs, so they can't be passed alongside the text prompt to HF processor. Therefore, we process the text and multi-modal inputs separately, using [dummy text](#dummy-text) to avoid HF errors. Since this skips HF's prompt updating code, we apply [automatic prompt updating](#automatic-prompt-updating) afterwards to keep the output tokens and multi-modal data consistent with each other.
## Speeding Up Multi‑Modal Data Processing ## Speeding Up Multi‑Modal Data Processing
### Fused Normalisation on the Device ### Fused Normalisation on the Device
+1 -1
View File
@@ -7,7 +7,7 @@ requests >= 2.26.0
tqdm tqdm
blake3 blake3
py-cpuinfo py-cpuinfo
transformers >= 5.5.3 transformers >= 5.10.4
huggingface_hub >= 1.28.0 huggingface_hub >= 1.28.0
tokenizers >= 0.21.1 # Required for fast incremental detokenization. tokenizers >= 0.21.1 # Required for fast incremental detokenization.
safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611 safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611
@@ -136,7 +136,7 @@ def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
# no audio track" case. (The audio limit above is 1, not 0, so that # no audio track" case. (The audio limit above is 1, not 0, so that
# `mm_counts["audio"]` is still declared for this request; a 0 limit # `mm_counts["audio"]` is still declared for this request; a 0 limit
# would make vLLM omit the "audio" key from `mm_counts` entirely, which # would make vLLM omit the "audio" key from `mm_counts` entirely, which
# trips an unrelated assertion in `_apply_hf_processor_mm_only`.) # trips an unrelated assertion in `_apply_hf_processor_main`.)
mm_data = {"video": [video]} mm_data = {"video": [video]}
with pytest.raises(ValueError, match="doesn't have audio track"): with pytest.raises(ValueError, match="doesn't have audio track"):
@@ -105,9 +105,9 @@ def test_audio_chunk_counting(mock_ctx):
audio_2 = np.zeros(75 * sr) audio_2 = np.zeros(75 * sr)
mm_data = {"audio": [audio_1, audio_2]} mm_data = {"audio": [audio_1, audio_2]}
prompt = "<|user|>Listen.<|end|>"
processed = processor._call_hf_processor(prompt, mm_data, {}) mm_items = processor.info.parse_mm_data(mm_data, validate=False)
processed = processor._apply_hf_processor_main(mm_items, {})
chunk_counts = processed["chunk_counts"] chunk_counts = processed["chunk_counts"]
@@ -385,17 +385,19 @@ def test_processing_correctness(
num_batches: int, num_batches: int,
simplify_rate: float, simplify_rate: float,
): ):
if model_id == "google/gemma-3n-E2B-it":
pytest.skip("Fix later")
if model_id == "OpenGVLab/InternVL2-2B":
pytest.skip("Fix later")
if model_id == "openvla/openvla-7b": if model_id == "openvla/openvla-7b":
pytest.skip( pytest.skip(
"OpenVLA uses a custom vLLM processor because its HF remote " "OpenVLA uses a custom vLLM processor because its HF remote "
"processor is incompatible with current Transformers." "processor is incompatible with current Transformers."
) )
if model_id == "jinaai/jina-reranker-m0": if model_id == "jinaai/jina-reranker-m0":
pytest.skip("Fix later") pytest.skip(
"JinaVL reverses the multi-modal item order inside "
"_apply_hf_processor_main to match its document-before-query "
"score template, which breaks the positional correspondence "
"between multi-modal kwargs and hashes assumed by the "
"processing cache. Needs a fix that keeps both consistent."
)
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602": if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
pytest.skip( pytest.skip(
"Voxtral Realtime doesn't make use of any place-holder " "Voxtral Realtime doesn't make use of any place-holder "
@@ -403,22 +405,12 @@ def test_processing_correctness(
"correctness test as is. Let's revisit adapting this " "correctness test as is. Let's revisit adapting this "
"test once more realtime models exist." "test once more realtime models exist."
) )
if model_id == "CohereLabs/cohere-transcribe-03-2026":
pytest.skip("Fix later")
if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"): if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"):
pytest.skip( pytest.skip(
"MOSS-Audio uses a custom processor that dynamically expands " "MOSS-Audio uses a custom processor that dynamically expands "
"audio placeholders from processed audio lengths. Its vLLM " "audio placeholders from processed audio lengths. Its vLLM "
"processor paths are covered by test_moss_audio.py." "processor paths are covered by test_moss_audio.py."
) )
# TODO: Remove when transformers 5.15.0 is released, which contains
# https://github.com/huggingface/transformers/pull/47483.
if model_id == "microsoft/VibeVoice-ASR-HF":
pytest.skip(
"VibeVoice ASR requires audio as a positional argument and hence "
"cannot pass the processing correctness test as is. Its generation "
"is covered by test_transformers_audio.py."
)
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct": if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
pytest.skip( pytest.skip(
"LLaVA-OneVision-2 video processing routes frames through custom " "LLaVA-OneVision-2 video processing routes frames through custom "
@@ -202,7 +202,7 @@ def test_validate_relative_bare_path_blocked():
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps) # Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps)
# #
# Non-codec videos reach _call_hf_processor as a ``(frames_ndarray, metadata)`` # Non-codec videos reach _apply_hf_processor_main as a ``(frames_ndarray, metadata)``
# tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real # tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real
# ``video_url`` inputs, or by the dummy-inputs builder during profiling # ``video_url`` inputs, or by the dummy-inputs builder during profiling
# (``video_needs_metadata=True``). The helper materialises PIL frames and # (``video_needs_metadata=True``). The helper materialises PIL frames and
@@ -103,7 +103,7 @@ def test_processor_multi_video(
"""Verify that multi-video processing produces correct placeholders. """Verify that multi-video processing produces correct placeholders.
This exercises the token-level replacement path in This exercises the token-level replacement path in
``_call_hf_processor`` which avoids the quadratic text-level ``_apply_hf_processor_main`` which avoids the quadratic text-level
prompt expansion. prompt expansion.
""" """
ctx = build_model_context( ctx = build_model_context(
@@ -153,7 +153,7 @@ def test_processor_multi_video_list_kwargs(
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not ``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
crash. crash.
Before the fix, ``_call_hf_processor`` copied the whole kwargs to every Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
video without slicing, so ``_get_video_second_idx`` received the list video without slicing, so ``_get_video_second_idx`` received the list
where a scalar was expected and raised ``TypeError``. where a scalar was expected and raised ``TypeError``.
""" """
+27 -20
View File
@@ -377,35 +377,41 @@ class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):
class AudioFlamingo3MultiModalProcessor( class AudioFlamingo3MultiModalProcessor(
BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo] BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo]
): ):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any], hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> tuple[Mapping[str, object], Mapping[str, object]]:
processor_mm_data = dict(mm_data) mm_data = dict(mm_data)
audios = processor_mm_data.pop("audios", None) audios = mm_data.pop("audios", None)
if audios is not None: if audios is not None:
processor_mm_data["audio"] = audios mm_data["audio"] = audios
outputs = super()._call_hf_processor( return mm_data, hf_processor_mm_kwargs
prompt=prompt,
mm_data=processor_mm_data,
mm_kwargs=mm_kwargs,
)
if "input_features_mask" in outputs: def _postprocess_hf_mm_data(
outputs["feature_attention_mask"] = outputs.pop("input_features_mask") self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "input_features_mask" in processed_data:
processed_data["feature_attention_mask"] = processed_data.pop(
"input_features_mask"
)
audio_data = processor_mm_data.get("audio") audio_data = mm_data.get("audio")
if audio_data is None: if audio_data is None:
return outputs return processed_data
audio_list = audio_data if isinstance(audio_data, list) else [audio_data] audio_list = audio_data if isinstance(audio_data, list) else [audio_data]
if len(audio_list) == 0: if len(audio_list) == 0:
return outputs return processed_data
processor = self.info.get_hf_processor(**mm_kwargs) processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
feature_extractor = processor.feature_extractor feature_extractor = processor.feature_extractor
sampling_rate = feature_extractor.sampling_rate sampling_rate = feature_extractor.sampling_rate
chunk_length = feature_extractor.chunk_length chunk_length = feature_extractor.chunk_length
@@ -423,8 +429,9 @@ class AudioFlamingo3MultiModalProcessor(
n_win = max_windows n_win = max_windows
chunk_counts.append(n_win) chunk_counts.append(n_win)
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long) processed_data["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
return outputs
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
-18
View File
@@ -467,24 +467,6 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]): class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
# HF processor always adds placeholders even when there's no image
tokenizer = self.info.get_tokenizer()
prompt_ids = tokenizer.encode(prompt)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
return super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
hf_inputs: BatchFeature, hf_inputs: BatchFeature,
+6 -24
View File
@@ -135,34 +135,16 @@ class ChameleonDummyInputsBuilder(BaseDummyInputsBuilder[ChameleonProcessingInfo
class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingInfo]): class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, # ChameleonProcessor requires text corresponding to the images
prompt: str, return self.dummy_inputs.get_dummy_text(mm_counts)
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
return super()._call_hf_processor( def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _apply_hf_processor_tokens_only(
self,
prompt_tokens: list[int],
) -> list[int]:
# HF processor adds sep token for chat mode # HF processor adds sep token for chat mode
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab() sep_token_id = tokenizer.get_vocab()[tokenizer.sep_token] # type: ignore
sep_token_id = vocab[tokenizer.sep_token] # type: ignore return [*prompt, sep_token_id]
return prompt_tokens + [sep_token_id]
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
-9
View File
@@ -16,7 +16,6 @@ import torch
import torch.nn as nn import torch.nn as nn
import torch.nn.functional as F import torch.nn.functional as F
from einops import rearrange from einops import rearrange
from transformers import BatchFeature
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import BaseDummyOptions
@@ -492,14 +491,6 @@ class CheersDummyInputsBuilder(BaseDummyInputsBuilder[CheersProcessingInfo]):
class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]): class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):
"""Multimodal processor for Cheers model.""" """Multimodal processor for Cheers model."""
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates( def _get_prompt_updates(
self, self,
mm_items: MultiModalDataItems, mm_items: MultiModalDataItems,
+13 -14
View File
@@ -222,24 +222,23 @@ class Cohere2VisionDummyInputsBuilder(
class Cohere2VisionMultiModalProcessor( class Cohere2VisionMultiModalProcessor(
BaseMultiModalProcessor[Cohere2VisionProcessingInfo] BaseMultiModalProcessor[Cohere2VisionProcessingInfo]
): ):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature: ) -> BatchFeature:
processed_outputs = super()._call_hf_processor( if not mm_data:
prompt, return processed_data
mm_data,
mm_kwargs,
)
# Ensure num_patches is available for proper tensor splitting
if ( if (
"num_patches" not in processed_outputs "num_patches" not in processed_data
and (images := mm_data.get("images")) is not None and (images := mm_data.get("images")) is not None
): ):
hf_processor = self.info.get_hf_processor(**mm_kwargs) hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
# Fallback calculation if HF processor didn't provide num_patches # Fallback calculation if HF processor didn't provide num_patches
mm_items = self.info.parse_mm_data({"image": images}, validate=False) mm_items = self.info.parse_mm_data({"image": images}, validate=False)
@@ -250,13 +249,13 @@ class Cohere2VisionMultiModalProcessor(
image_width=parsed_images.get_image_size(i).width, image_width=parsed_images.get_image_size(i).width,
image_height=parsed_images.get_image_size(i).height, image_height=parsed_images.get_image_size(i).height,
processor=hf_processor, processor=hf_processor,
mm_kwargs=mm_kwargs, mm_kwargs=hf_processor_mm_kwargs,
) )
for i in range(len(parsed_images)) for i in range(len(parsed_images))
] ]
processed_outputs["num_patches"] = torch.tensor(num_patches) processed_data["num_patches"] = torch.tensor(num_patches)
return processed_outputs return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+46 -19
View File
@@ -8,7 +8,7 @@ from typing import Any, ClassVar
import torch import torch
import torch.nn.functional as F import torch.nn.functional as F
from torch import nn from torch import nn
from transformers import PretrainedConfig from transformers import BatchFeature, PretrainedConfig
from vllm.compilation.decorators import support_torch_compile from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
@@ -45,9 +45,12 @@ from vllm.multimodal.processing import (
BaseDummyInputsBuilder, BaseDummyInputsBuilder,
BaseProcessingInfo, BaseProcessingInfo,
EncDecMultiModalProcessor, EncDecMultiModalProcessor,
ProcessorInputs,
PromptReplacement, PromptReplacement,
PromptUpdate, PromptUpdate,
TimingContext,
) )
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
from vllm.renderers import TokenizeParams from vllm.renderers import TokenizeParams
from vllm.tokenizers import cached_tokenizer_from_config from vllm.tokenizers import cached_tokenizer_from_config
from vllm.transformers_utils.processors.cohere_asr import ( from vllm.transformers_utils.processors.cohere_asr import (
@@ -1941,27 +1944,51 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
) -> list[int]: ) -> list[int]:
return [0] return [0]
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
): ) -> tuple[Mapping[str, object], Mapping[str, object]]:
if mm_data: feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_data = dict(audio=dict(mm_data).pop("audios")) mm_data = dict(mm_data)
mm_kwargs = dict( mm_data["audio"] = mm_data.pop("audios")
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate, hf_processor_mm_kwargs = dict(
) **hf_processor_mm_kwargs,
processed_outputs = super()._call_hf_processor( sampling_rate=feature_extractor.sampling_rate,
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
) )
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels") return mm_data, hf_processor_mm_kwargs
return processed_outputs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
# Dithering injects noise into the extracted features, so the
# feature extractor is not a pure function of its input. Since the
# processing cache assumes that processor outputs are invariant
# across calls, bypass the cache when dithering is active.
preproc = self.info.get_hf_config().preprocessor
if preproc.get("dither", 1e-05) > 0:
return self._apply_hf_processor(inputs, timing_ctx)
return super()._cached_apply_hf_processor(inputs, timing_ctx)
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+17 -6
View File
@@ -154,16 +154,25 @@ class ColModernVBertDummyInputsBuilder(
class ColModernVBertMultiModalProcessor( class ColModernVBertMultiModalProcessor(
BaseMultiModalProcessor[ColModernVBertProcessingInfo], BaseMultiModalProcessor[ColModernVBertProcessingInfo],
): ):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer) assert isinstance(tokenizer, HfTokenizer)
text_encoding = tokenizer( text_encoding = tokenizer(
prompt, prompt_text,
return_tensors="pt", return_tensors="pt",
) )
result = BatchFeature(data=dict(text_encoding)) result = BatchFeature(data=dict(text_encoding))
@@ -183,7 +192,9 @@ class ColModernVBertMultiModalProcessor(
) )
result.update(image_outputs) result.update(image_outputs)
return result processed_data = result
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+24 -16
View File
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import MultiModalDataItems
from .interfaces import SupportsLateInteraction from .interfaces import SupportsLateInteraction
from .interfaces_base import default_pooling_type from .interfaces_base import default_pooling_type
@@ -59,24 +60,31 @@ class ColPaliProcessingInfo(PaliGemmaProcessingInfo):
class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor): class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
"""Multimodal processor for ColPali.""" """Multimodal processor for ColPali."""
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
if mm_data: valid_mm_items = mm_items.select(
# The ColPali tokenizer_config.json ships with a small default {k for k, c in mm_items.get_all_counts().items() if c > 0}
# max_length (50) that truncates the 1024 image tokens inserted )
# by PaliGemmaProcessor, causing a token-count mismatch. mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
# vLLM enforces its own max_model_len, so we disable HF
# truncation to keep all image + text tokens intact. if not mm_data:
return self.info.ctx.call_hf_processor( return BatchFeature(dict(passthrough_data))
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data), # The ColPali tokenizer_config.json ships with a small default
dict(**mm_kwargs, truncation=False), # max_length (50) that truncates the 1024 image tokens inserted
) # by PaliGemmaProcessor, causing a token-count mismatch.
return super()._call_hf_processor(prompt, mm_data, mm_kwargs) # vLLM enforces its own max_model_len, so we disable HF
# truncation to keep all image + text tokens intact.
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
dict(**hf_processor_mm_kwargs, truncation=False),
)
processed_data.update(passthrough_data)
return processed_data
@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL") @default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
+18 -11
View File
@@ -287,27 +287,34 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing
class DeepseekOCRMultiModalProcessor( class DeepseekOCRMultiModalProcessor(
BaseMultiModalProcessor[DeepseekOCRProcessingInfo] BaseMultiModalProcessor[DeepseekOCRProcessingInfo]
): ):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
if mm_data: if mm_data:
processed_outputs = self.info.ctx.call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs), self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(prompt=prompt, **mm_data), dict(prompt=prompt_text, **mm_data),
mm_kwargs, hf_processor_mm_kwargs,
) )
else: else:
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer) assert isinstance(tokenizer, HfTokenizer)
processed_outputs = tokenizer( processed_data = tokenizer(
prompt, add_special_tokens=True, return_tensors="pt" prompt_text, add_special_tokens=True, return_tensors="pt"
) )
return processed_outputs processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+19 -11
View File
@@ -158,27 +158,35 @@ class DeepseekOCR2DummyInputsBuilder(
class DeepseekOCR2MultiModalProcessor( class DeepseekOCR2MultiModalProcessor(
BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo] BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo]
): ):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
if mm_data: if mm_data:
processed_outputs = self.info.ctx.call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs), self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(prompt=prompt, **mm_data), dict(prompt=prompt_text, **mm_data),
mm_kwargs, hf_processor_mm_kwargs,
) )
else: else:
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer) assert isinstance(tokenizer, HfTokenizer)
processed_outputs = tokenizer( processed_data = tokenizer(
prompt, add_special_tokens=True, return_tensors="pt" prompt_text, add_special_tokens=True, return_tensors="pt"
) )
return processed_outputs processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+11 -17
View File
@@ -43,7 +43,6 @@ from vllm.multimodal.processing.processor import (
) )
from vllm.sequence import IntermediateTensors from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.hf import HfTokenizer
from vllm.transformers_utils.configs.deepseek_vl2 import ( from vllm.transformers_utils.configs.deepseek_vl2 import (
DeepseekVLV2Config, DeepseekVLV2Config,
MlpProjectorConfig, MlpProjectorConfig,
@@ -236,28 +235,23 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
class DeepseekVL2MultiModalProcessor( class DeepseekVL2MultiModalProcessor(
BaseMultiModalProcessor[DeepseekVL2ProcessingInfo] BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]
): ):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature: ) -> BatchFeature:
if not mm_data: if not mm_data:
tokenizer = self.info.get_tokenizer() return processed_data
assert isinstance(tokenizer, HfTokenizer)
return tokenizer(prompt, add_special_tokens=True, return_tensors="pt")
processed_outputs = super()._call_hf_processor( processed_data["num_patches"] = (
prompt=prompt, processed_data["images_spatial_crop"].prod(-1) + 1
mm_data=mm_data,
mm_kwargs=mm_kwargs,
) )
processed_outputs["num_patches"] = ( return processed_data
processed_outputs["images_spatial_crop"].prod(-1) + 1
)
return processed_outputs
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
@@ -315,7 +309,7 @@ class DeepseekVL2MultiModalProcessor(
self, self,
inputs: ProcessorInputs, inputs: ProcessorInputs,
timing_ctx: TimingContext, timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]: ) -> MultiModalProcessingInfo:
# The processor logic is different for len(images) <= 2 vs > 2 # The processor logic is different for len(images) <= 2 vs > 2
# Since the processing cache assumes that the processor output is # Since the processing cache assumes that the processor output is
# invariant of how many images are passed per prompt, we only # invariant of how many images are passed per prompt, we only
+21 -18
View File
@@ -1110,21 +1110,20 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
pixel_values = pixel_values.to(hf_config.dtype) pixel_values = pixel_values.to(hf_config.dtype)
return pixel_values return pixel_values
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
# when the prompt is not empty but the multimodal data is empty, valid_mm_items = mm_items.select(
# directly invoke the tokenizer. {k for k, c in mm_items.get_all_counts().items() if c > 0}
if "images" not in mm_data and "videos" not in mm_data and prompt != "": )
tokenizer = self.info.get_tokenizer() mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_ids = tokenizer.encode(prompt)
tokenizer_output = BatchFeature( if "images" not in mm_data and "videos" not in mm_data:
dict(input_ids=[prompt_ids]), tensor_type="pt" return BatchFeature(dict(passthrough_data))
)
return tokenizer_output prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
if "images" not in mm_data: if "images" not in mm_data:
mm_data["images"] = [] mm_data["images"] = []
@@ -1132,7 +1131,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
mm_data["videos"] = [] mm_data["videos"] = []
# Check if HF processor supports video metadata # Check if HF processor supports video metadata
hf_processor = self.info.get_hf_processor(**mm_kwargs) hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
supports_video_metadata = getattr( supports_video_metadata = getattr(
hf_processor, "supports_video_metadata", False hf_processor, "supports_video_metadata", False
) )
@@ -1149,8 +1148,10 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
processor_output = self.info.ctx.call_hf_processor( processor_output = self.info.ctx.call_hf_processor(
hf_processor, hf_processor,
dict(text=[prompt], images=mm_data["images"], videos=mm_data["videos"]), dict(
mm_kwargs, text=[prompt_text], images=mm_data["images"], videos=mm_data["videos"]
),
hf_processor_mm_kwargs,
) )
# Divide the processor_output into two modalities: image and video. # Divide the processor_output into two modalities: image and video.
@@ -1158,7 +1159,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
pixel_values = processor_output["images"] pixel_values = processor_output["images"]
if pixel_values is not None: if pixel_values is not None:
processor_output["images"] = self._pixel_values_norm( processor_output["images"] = self._pixel_values_norm(
pixel_values, mm_kwargs pixel_values, hf_processor_mm_kwargs
) )
for key in list(processor_output.keys()): for key in list(processor_output.keys()):
if processor_output[key] is None: if processor_output[key] is None:
@@ -1184,7 +1185,9 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
] ]
del processor_output["images"] del processor_output["images"]
return processor_output processed_data = processor_output
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates( def _get_prompt_updates(
self, self,
+27 -18
View File
@@ -228,27 +228,36 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing
class FireRedASR2MultiModalProcessor( class FireRedASR2MultiModalProcessor(
BaseMultiModalProcessor[FireRedASR2ProcessingInfo] BaseMultiModalProcessor[FireRedASR2ProcessingInfo]
): ):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> tuple[Mapping[str, object], Mapping[str, object]]:
if mm_data: feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_data = dict(audio=mm_data.pop("audios")) mm_data = dict(mm_data)
mm_kwargs = dict( mm_data["audio"] = mm_data.pop("audios")
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate, hf_processor_mm_kwargs = dict(
) **hf_processor_mm_kwargs,
processed_outputs = super()._call_hf_processor( sampling_rate=feature_extractor.sampling_rate,
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
) )
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels") return mm_data, hf_processor_mm_kwargs
return processed_outputs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+27 -18
View File
@@ -451,27 +451,36 @@ class FireRedLIDMultiModalProcessor(
# Dummy encoder prompt for profiling (encoder only processes audio). # Dummy encoder prompt for profiling (encoder only processes audio).
return [0] return [0]
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> tuple[Mapping[str, object], Mapping[str, object]]:
if mm_data: feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_data = dict(audio=mm_data.pop("audios")) mm_data = dict(mm_data)
mm_kwargs = dict( mm_data["audio"] = mm_data.pop("audios")
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate, hf_processor_mm_kwargs = dict(
) **hf_processor_mm_kwargs,
processed_outputs = super()._call_hf_processor( sampling_rate=feature_extractor.sampling_rate,
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
) )
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels") return mm_data, hf_processor_mm_kwargs
return processed_outputs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+44 -18
View File
@@ -44,9 +44,12 @@ from vllm.multimodal.processing import (
BaseDummyInputsBuilder, BaseDummyInputsBuilder,
BaseMultiModalProcessor, BaseMultiModalProcessor,
BaseProcessingInfo, BaseProcessingInfo,
ProcessorInputs,
PromptReplacement, PromptReplacement,
PromptUpdate, PromptUpdate,
TimingContext,
) )
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
from vllm.transformers_utils.processor import cached_processor_from_config from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor
from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -747,27 +750,50 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]): class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> tuple[Mapping[str, object], Mapping[str, object]]:
if mm_data: feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_data = dict(audio=mm_data.pop("audios")) mm_data = dict(mm_data)
mm_kwargs = dict( mm_data["audio"] = mm_data.pop("audios")
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate, hf_processor_mm_kwargs = dict(
) **hf_processor_mm_kwargs,
processed_outputs = super()._call_hf_processor( sampling_rate=feature_extractor.sampling_rate,
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
) )
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels") return mm_data, hf_processor_mm_kwargs
return processed_outputs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
# Dithering injects noise into the extracted features, so the
# feature extractor is not a pure function of its input. Since the
# processing cache assumes that processor outputs are invariant
# across calls, bypass the cache when dithering is active.
if self.info.get_feature_extractor().dither > 0:
return self._apply_hf_processor(inputs, timing_ctx)
return super()._cached_apply_hf_processor(inputs, timing_ctx)
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+21 -8
View File
@@ -612,20 +612,31 @@ class FunAudioChatDummyInputsBuilder(
class FunAudioChatMultiModalProcessor( class FunAudioChatMultiModalProcessor(
BaseMultiModalProcessor[FunAudioChatProcessingInfo] BaseMultiModalProcessor[FunAudioChatProcessingInfo]
): ):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
input_ids = torch.tensor([tokenizer.encode(prompt)]) input_ids = torch.tensor([tokenizer.encode(prompt_text)])
audios = mm_data.get("audios", []) audios = mm_data.get("audios", [])
if not audios: if not audios:
return BatchFeature({"input_ids": input_ids}) processed_data = BatchFeature({"input_ids": input_ids})
processed_data.update(passthrough_data)
return processed_data
feature_extractor = self.info.get_feature_extractor(**mm_kwargs) feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
sr = int(feature_extractor.sampling_rate) sr = int(feature_extractor.sampling_rate)
min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400) min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400)
@@ -676,7 +687,9 @@ class FunAudioChatMultiModalProcessor(
"feature_exist_mask": torch.ones((len(wavs),), dtype=torch.bool), "feature_exist_mask": torch.ones((len(wavs),), dtype=torch.bool),
} }
return BatchFeature({"input_ids": input_ids, **mm_inputs}) processed_data = BatchFeature({"input_ids": input_ids, **mm_inputs})
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+12 -13
View File
@@ -261,39 +261,38 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]): class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature: ) -> BatchFeature:
processed_outputs = super()._call_hf_processor( if not mm_data:
prompt, return processed_data
mm_data,
mm_kwargs,
)
# HF processor pops the `num_crops` kwarg, which is needed by vLLM
if (images := mm_data.get("images")) is not None: if (images := mm_data.get("images")) is not None:
mm_items = self.info.parse_mm_data({"image": images}, validate=False) mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems) parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [ image_sizes = [
parsed_images.get_image_size(i) for i in range(len(parsed_images)) parsed_images.get_image_size(i) for i in range(len(parsed_images))
] ]
hf_processor = self.info.get_hf_processor(**mm_kwargs) hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
num_crops = [ num_crops = [
self.info.get_num_crops( self.info.get_num_crops(
image_width=size.width, image_width=size.width,
image_height=size.height, image_height=size.height,
processor=hf_processor, processor=hf_processor,
mm_kwargs=mm_kwargs, mm_kwargs=hf_processor_mm_kwargs,
) )
for size in image_sizes for size in image_sizes
] ]
processed_outputs["num_patches"] = torch.tensor(num_crops) + 1 processed_data["num_patches"] = torch.tensor(num_crops) + 1
return processed_outputs return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+57 -27
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import inspect
from collections.abc import Iterable, Mapping, Sequence from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal from typing import Annotated, Any, Literal
@@ -257,40 +258,69 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]): class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> tuple[Mapping[str, object], Mapping[str, object]]:
# HF Transformers audio processor no longer accepts `audios` key. mm_data = dict(mm_data)
# We pop `audios` and replace it with `audio` key to suppress
# the warning.
if "audios" in mm_data: if "audios" in mm_data:
mm_data["audio"] = mm_data.pop("audios") mm_data["audio"] = mm_data.pop("audios")
processed_outputs = super()._call_hf_processor(
prompt,
mm_data,
mm_kwargs,
)
if "input_features" in processed_outputs: return mm_data, hf_processor_mm_kwargs
# Padding enables audio_tower to run in batched mode
processed_outputs["input_features_padded"] = processed_outputs[ def _postprocess_hf_mm_data(
"input_features" self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "input_features" in processed_data:
# The feature extractor pads every clip to the longest one in
# the batch, so each clip's frame count depends on the other
# clips it was processed with. Trim each clip to the number of
# frames it would have if processed alone so that each item's
# output (and thus its cache entry) is independent of the other
# items in the batch. `batch_audio_features` re-pads the
# per-item features so audio_tower can still run batched.
num_frames = [
self._get_num_audio_frames_alone(len(audio))
for audio in mm_data["audio"]
]
processed_data["input_features_padded"] = [
features[:n]
for features, n in zip(processed_data.pop("input_features"), num_frames)
]
processed_data["input_features_mask"] = [
mask[:n]
for mask, n in zip(processed_data["input_features_mask"], num_frames)
] ]
# Unpad features here since we need the output of each item to be return processed_data
# independent of other items for the cache to work correctly
unpadded_features = [ def _get_num_audio_frames_alone(self, num_samples: int) -> int:
f[mask] """
for f, mask in zip( Get the number of mel frames that the feature extractor produces
processed_outputs["input_features"], for a single audio clip of the given length, independent of batch
processed_outputs["input_features_mask"], padding.
) """
] feature_extractor = self.info.get_feature_extractor()
processed_outputs["input_features"] = unpadded_features call_params = inspect.signature(type(feature_extractor).__call__).parameters
return processed_outputs max_length = call_params["max_length"].default
pad_to_multiple_of = call_params["pad_to_multiple_of"].default
num_samples = min(num_samples, max_length)
remainder = num_samples % pad_to_multiple_of
if remainder:
num_samples += pad_to_multiple_of - remainder
frame_size = feature_extractor.frame_length + 1
num_frames = (num_samples - frame_size) // feature_extractor.hop_length + 1
return max(num_frames, 0)
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+38 -28
View File
@@ -560,13 +560,22 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs) valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs) val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
if val is not None and val not in _SUPPORTED_SOFT_TOKENS: if val is not None and val not in _SUPPORTED_SOFT_TOKENS:
@@ -611,15 +620,15 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
timestamps = [idx / fps for idx in frame_indices] timestamps = [idx / fps for idx in frame_indices]
# Process frames as images with max_soft_tokens=70 # Process frames as images with max_soft_tokens=70
video_mm_kwargs = dict(mm_kwargs) video_mm_kwargs = dict(hf_processor_mm_kwargs)
video_mm_kwargs["max_soft_tokens"] = _VIDEO_MAX_SOFT_TOKENS video_mm_kwargs["max_soft_tokens"] = _VIDEO_MAX_SOFT_TOKENS
dummy_prompt = ("\t" + processor.image_token) * len(frames) dummy_prompt = ("\t" + processor.image_token) * len(frames)
frame_outputs = super()._call_hf_processor( frame_outputs = self.info.ctx.call_hf_processor(
prompt=dummy_prompt, self.info.get_hf_processor(**video_mm_kwargs),
mm_data={"images": frames}, dict(text=dummy_prompt, **{"images": frames}),
mm_kwargs=video_mm_kwargs, video_mm_kwargs,
) )
# Remap HF key name # Remap HF key name
@@ -660,7 +669,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
# split-replace bug where replacement text (which itself # split-replace bug where replacement text (which itself
# contains <|video|> tokens) collides with later splits. # contains <|video|> tokens) collides with later splits.
vt = processor.video_token vt = processor.video_token
parts = prompt.split(vt, len(video_replacements)) parts = prompt_text.split(vt, len(video_replacements))
# NOTE: len(parts) <= len(video_replacements) + 1 # NOTE: len(parts) <= len(video_replacements) + 1
parts_with_repl: list[str] = [] parts_with_repl: list[str] = []
@@ -668,7 +677,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
parts_with_repl.extend([part, repl]) parts_with_repl.extend([part, repl])
parts_with_repl.extend(parts[len(video_replacements) :]) parts_with_repl.extend(parts[len(video_replacements) :])
prompt = "".join(parts_with_repl) prompt_text = "".join(parts_with_repl)
video_outputs = { video_outputs = {
"pixel_values_videos": torch.cat(all_video_pixel_values, dim=0), "pixel_values_videos": torch.cat(all_video_pixel_values, dim=0),
@@ -712,45 +721,46 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
# NOTE: This requires a corresponding type annotation on the # NOTE: This requires a corresponding type annotation on the
# HF side (Gemma4ProcessorKwargs.images_kwargs) so that # HF side (Gemma4ProcessorKwargs.images_kwargs) so that
# _merge_kwargs routes max_soft_tokens into images_kwargs. # _merge_kwargs routes max_soft_tokens into images_kwargs.
patched_mm_kwargs = dict(mm_kwargs) patched_mm_kwargs = dict(hf_processor_mm_kwargs)
if val is not None and is_top_level_max_soft_tokens: if val is not None and is_top_level_max_soft_tokens:
patched_mm_kwargs["max_soft_tokens"] = val patched_mm_kwargs["max_soft_tokens"] = val
processed_outputs = super()._call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
prompt, self.info.get_hf_processor(**patched_mm_kwargs),
mm_data, dict(text=prompt_text, **mm_data),
patched_mm_kwargs, patched_mm_kwargs,
) )
# HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'. # HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'.
# Remap here to keep a single translation point. # Remap here to keep a single translation point.
if "image_position_ids" in processed_outputs: if "image_position_ids" in processed_data:
processed_outputs["pixel_position_ids"] = processed_outputs.pop( processed_data["pixel_position_ids"] = processed_data.pop(
"image_position_ids" "image_position_ids"
) )
if "input_features" in processed_outputs: if "input_features" in processed_data:
# Unpad per-item so each item's cache entry is # Unpad per-item so each item's cache entry is
# self-contained. The batched() field config in # self-contained. The batched() field config in
# _get_mm_fields_config stacks equal-length items and leaves # _get_mm_fields_config stacks equal-length items and leaves
# the rest as a list; _process_audio_input re-pads that list # the rest as a list; _process_audio_input re-pads that list
# to the batch's max length. # to the batch's max length.
masks = processed_outputs["input_features_mask"] masks = processed_data["input_features_mask"]
unpadded_features = [ unpadded_features = [
f[mask] f[mask]
for f, mask in zip( for f, mask in zip(
processed_outputs["input_features"], processed_data["input_features"],
masks, masks,
) )
] ]
unpadded_masks = [mask[mask] for mask in masks] unpadded_masks = [mask[mask] for mask in masks]
processed_outputs["input_features"] = unpadded_features processed_data["input_features"] = unpadded_features
processed_outputs["input_features_padded"] = unpadded_features processed_data["input_features_padded"] = unpadded_features
processed_outputs["input_features_mask"] = unpadded_masks processed_data["input_features_mask"] = unpadded_masks
# Merge video outputs into the final result processed_data.update(video_outputs)
combined_outputs = dict(processed_outputs, **video_outputs) processed_data.update(passthrough_data)
return BatchFeature(combined_outputs)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
@@ -815,7 +825,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
image_size = images.get_image_size(item_idx) image_size = images.get_image_size(item_idx)
# Resolve the effective max_soft_tokens by merging # Resolve the effective max_soft_tokens by merging
# per-prompt kwargs with the config-level defaults, # per-prompt kwargs with the config-level defaults,
# consistent with how _call_hf_processor resolves it. # consistent with how _apply_hf_processor_main resolves it.
# Without this merge, a missing per-prompt override # Without this merge, a missing per-prompt override
# would fall back to vision_cfg.default_output_length # would fall back to vision_cfg.default_output_length
# instead of the config's mm_processor_kwargs default. # instead of the config's mm_processor_kwargs default.
+42 -32
View File
@@ -1576,32 +1576,39 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
return prepared_data, prepared_kwargs return prepared_data, prepared_kwargs
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
mm_data = dict(mm_data) valid_mm_items = mm_items.select(
if not mm_data: {k for k, c in mm_items.get_all_counts().items() if c > 0}
tokenizer = self.info.get_tokenizer() )
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False) mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
processor = self.info.get_hf_processor(**mm_kwargs) if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data)
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
use_direct_path = ( use_direct_path = (
not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
) )
if use_direct_path: if use_direct_path:
prepared_data, prepared_kwargs = self._get_direct_path_inputs( prepared_data, prepared_kwargs = self._get_direct_path_inputs(
mm_data, mm_kwargs mm_data, hf_processor_mm_kwargs
) )
return super()._call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
prompt=prompt, self.info.get_hf_processor(**prepared_kwargs),
mm_data=prepared_data, dict(text=prompt_text, **prepared_data),
mm_kwargs=prepared_kwargs, prepared_kwargs,
) )
processed_data.update(passthrough_data)
return processed_data
if ( if (
"videos" in mm_data "videos" in mm_data
@@ -1616,7 +1623,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
video_array, metadata = item video_array, metadata = item
# don't update mm_kwargs inplace # don't update mm_kwargs inplace
video_mm_kwargs = dict(**mm_kwargs) video_mm_kwargs = dict(**hf_processor_mm_kwargs)
video_mm_kwargs["do_sample_frames"] = metadata.get( video_mm_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", True "do_sample_frames", True
) )
@@ -1625,10 +1632,13 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
video_mm_data["videos"] = [[video_array]] video_mm_data["videos"] = [[video_array]]
video_mm_data["video_metadata"] = [[_to_video_metadata(metadata)]] video_mm_data["video_metadata"] = [[_to_video_metadata(metadata)]]
video_outputs = super()._call_hf_processor( video_outputs = self.info.ctx.call_hf_processor(
prompt="<|begin_of_video|><|video|><|end_of_video|>", self.info.get_hf_processor(**video_mm_kwargs),
mm_data=video_mm_data, dict(
mm_kwargs=video_mm_kwargs, text="<|begin_of_video|><|video|><|end_of_video|>",
**video_mm_data,
),
video_mm_kwargs,
) )
input_ids = video_outputs.pop("input_ids") input_ids = video_outputs.pop("input_ids")
if swap_video_frame_tokens: if swap_video_frame_tokens:
@@ -1636,7 +1646,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
processor.video_token_id processor.video_token_id
) )
video_placeholder = processor.tokenizer.batch_decode(input_ids)[0] video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
prompt = prompt.replace( prompt_text = prompt_text.replace(
"<|begin_of_video|><|video|><|end_of_video|>", "<|begin_of_video|><|video|><|end_of_video|>",
video_placeholder, video_placeholder,
1, 1,
@@ -1652,20 +1662,20 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
video_outputs = dict() video_outputs = dict()
swap_video_frame_tokens = False swap_video_frame_tokens = False
processed_outputs = super()._call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
prompt=prompt, self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data=mm_data, dict(text=prompt_text, **mm_data),
mm_kwargs=mm_kwargs, hf_processor_mm_kwargs,
) )
if swap_video_frame_tokens: if swap_video_frame_tokens:
input_ids = processed_outputs["input_ids"] input_ids = processed_data["input_ids"]
input_ids[input_ids == processor.video_token_id] = processor.image_token_id input_ids[input_ids == processor.video_token_id] = processor.image_token_id
processed_outputs["input_ids"] = input_ids processed_data["input_ids"] = input_ids
combined_outputs = dict(
processed_outputs, processed_data.update(video_outputs)
**video_outputs, processed_data.update(passthrough_data)
)
return BatchFeature(combined_outputs) return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+37 -39
View File
@@ -750,46 +750,40 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
chunk_counts.append(min(n_chunks, max_windows)) chunk_counts.append(min(n_chunks, max_windows))
return chunk_counts return chunk_counts
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self, self,
prompt: str, mm_data: Mapping[str, object],
mm_data: dict[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, Any], ) -> tuple[Mapping[str, object], Mapping[str, object]]:
) -> BatchFeature: mm_data = dict(mm_data)
# Normalize input: handle deprecated key and list conversion.
if "audios" in mm_data: if "audios" in mm_data:
mm_data["audio"] = mm_data.pop("audios") mm_data["audio"] = mm_data.pop("audios")
audio = mm_data.get("audio", []) feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
audio_list = [audio] if audio and not isinstance(audio, list) else audio hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
# Early return for text-only.
if not audio_list:
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
# Handle sampling_rate
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_kwargs = dict(
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate, sampling_rate=feature_extractor.sampling_rate,
) )
# Call parent method return mm_data, hf_processor_mm_kwargs
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
# Postprocess: rename mask and add chunk counts def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
# Handle different key names from different transformers versions # Handle different key names from different transformers versions
if "input_features_mask" in outputs: if "input_features_mask" in processed_data:
outputs["feature_attention_mask"] = outputs.pop("input_features_mask") processed_data["feature_attention_mask"] = processed_data.pop(
elif "input_features_mask" not in outputs and "input_features" in outputs: "input_features_mask"
)
elif "input_features" in processed_data:
# If no mask is provided, create one from input_features # If no mask is provided, create one from input_features
input_features = outputs["input_features"] input_features = processed_data["input_features"]
if isinstance(input_features, torch.Tensor): if isinstance(input_features, torch.Tensor):
# Create a mask of all ones matching the sequence length # Create a mask of all ones matching the sequence length
mask = torch.ones( mask = torch.ones(
@@ -797,18 +791,22 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
input_features.shape[-1], input_features.shape[-1],
dtype=torch.long, dtype=torch.long,
) )
outputs["feature_attention_mask"] = mask processed_data["feature_attention_mask"] = mask
# Get processor for chunk counts calculation audio = mm_data.get("audio", [])
processor = self.info.get_hf_processor(**mm_kwargs) audio_list = [audio] if audio and not isinstance(audio, list) else audio
if audio_list:
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
# Override chunk counts calculation with GLM-ASR specific logic # Override chunk counts calculation with GLM-ASR specific logic
chunk_counts = self._calculate_chunk_counts( chunk_counts = self._calculate_chunk_counts(
audio_list, processor.feature_extractor, processor audio_list, processor.feature_extractor, processor
) )
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long) processed_data["chunk_counts"] = torch.tensor(
chunk_counts, dtype=torch.long
)
return outputs return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+16 -12
View File
@@ -180,12 +180,14 @@ class GraniteSpeechMultiModalProcessor(
) )
] ]
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data) mm_data = dict(mm_data)
audios = mm_data.pop("audios", []) audios = mm_data.pop("audios", [])
@@ -193,21 +195,23 @@ class GraniteSpeechMultiModalProcessor(
# GraniteSpeechFeatureExtractor accepts "audio" # GraniteSpeechFeatureExtractor accepts "audio"
mm_data["audio"] = audios mm_data["audio"] = audios
processed_outputs = super()._call_hf_processor( return mm_data, hf_processor_mm_kwargs
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "audio" in mm_data: if "audio" in mm_data:
# Calculate the number of audio tokens per entry in the batch; # Calculate the number of audio tokens per entry in the batch;
# This is used to split the batch back out after padding. # This is used to split the batch back out after padding.
audio_token_index = self.info.get_hf_config().audio_token_index audio_token_index = self.info.get_hf_config().audio_token_index
processed_outputs["audio_embed_sizes"] = ( processed_data["audio_embed_sizes"] = (
processed_outputs["input_ids"] == audio_token_index processed_data["input_ids"] == audio_token_index
).sum(-1) ).sum(-1)
return processed_outputs return processed_data
class GraniteSpeechDummyInputsBuilder( class GraniteSpeechDummyInputsBuilder(
+1 -1
View File
@@ -135,7 +135,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
self, self,
inputs: ProcessorInputs, inputs: ProcessorInputs,
timing_ctx: TimingContext, timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]: ) -> MultiModalProcessingInfo:
# The processor logic is different for len(images) <= 1 vs > 1 # The processor logic is different for len(images) <= 1 vs > 1
# Since the processing cache assumes that the processor output is # Since the processing cache assumes that the processor output is
# invariant of how many images are passed per prompt, we only # invariant of how many images are passed per prompt, we only
@@ -731,28 +731,6 @@ class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo
class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]): class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
hf_processor = self.info.get_hf_processor(**mm_kwargs)
# HunYuanVLProcessor requires image placeholders wrapped with start/end tokens.
if mm_data.get("images") is not None and prompt:
img_tok = hf_processor.image_token
wrapped = (
f"{hf_processor.image_start_token}{img_tok}"
f"{hf_processor.image_end_token}"
)
if img_tok in prompt and wrapped not in prompt:
prompt = prompt.replace(img_tok, wrapped)
return self.info.ctx.call_hf_processor(
hf_processor,
dict(text=prompt, **mm_data),
mm_kwargs,
)
def _get_prompt_updates( def _get_prompt_updates(
self, self,
mm_items: MultiModalDataItems, mm_items: MultiModalDataItems,
@@ -190,77 +190,71 @@ class HCXVisionDummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionProcessingInfo
class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingInfo]): class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingInfo]):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
for video_idx, video_arr in enumerate(mm_data.get("videos", [])): for video_idx, video_arr in enumerate(mm_data.get("videos", [])):
if video_arr.dtype != np.uint8: if video_arr.dtype != np.uint8:
mm_data["videos"][video_idx] = video_arr.astype(np.uint8) mm_data["videos"][video_idx] = video_arr.astype(np.uint8)
processed_outputs = self.info.ctx.call_hf_processor( images = mm_data.get("images")
hf_processor=self.info.get_hf_processor(**mm_kwargs), videos = mm_data.get("videos")
# batchify input as a single item
processed_data = self.info.ctx.call_hf_processor(
hf_processor=self.info.get_hf_processor(**hf_processor_mm_kwargs),
data=dict( data=dict(
text=prompt, images=None if images is None else [images],
images=None, videos=None if videos is None else [videos],
videos=None,
), ),
) # text-only )
if len(mm_data) > 0: for k, v in processed_data.items():
images = mm_data.get("images") if isinstance(v, list) and len(v) > 0:
videos = mm_data.get("videos") assert len(v) == 1
processed_data[k] = v[0]
# batchify input as a single item if images:
_processed_outputs = self.info.ctx.call_hf_processor( processed_data["image_sizes_images"] = torch.tensor(
hf_processor=self.info.get_hf_processor(**mm_kwargs), processed_data["image_sizes_images"]
data=dict( )
text=None, processed_data["vision_query_lengths_images"] = torch.tensor(
images=None if images is None else [images], processed_data["vision_query_lengths_images"]
videos=None if videos is None else [videos], )
),
) # mm-only
for k, v in _processed_outputs.items(): if videos:
if isinstance(v, list) and len(v) > 0: _idx_per_video = [
assert len(v) == 1 0,
_processed_outputs[k] = v[0] *accumulate(get_num_combined_frames(len(video)) for video in videos),
]
if images: processed_data["pixel_values_videos"] = [
_processed_outputs["image_sizes_images"] = torch.tensor( processed_data["pixel_values_videos"][
_processed_outputs["image_sizes_images"] _idx_per_video[i] : _idx_per_video[i + 1]
)
_processed_outputs["vision_query_lengths_images"] = torch.tensor(
_processed_outputs["vision_query_lengths_images"]
)
if videos:
_idx_per_video = [
0,
*accumulate(
get_num_combined_frames(len(video)) for video in videos
),
] ]
_processed_outputs["pixel_values_videos"] = [ for i in range(len(videos))
_processed_outputs["pixel_values_videos"][ ]
processed_data["vision_query_lengths_videos"] = [
torch.tensor(
processed_data["vision_query_lengths_videos"][
_idx_per_video[i] : _idx_per_video[i + 1] _idx_per_video[i] : _idx_per_video[i + 1]
] ]
for i in range(len(videos)) )
] for i in range(len(videos))
_processed_outputs["vision_query_lengths_videos"] = [ ]
torch.tensor(
_processed_outputs["vision_query_lengths_videos"][
_idx_per_video[i] : _idx_per_video[i + 1]
]
)
for i in range(len(videos))
]
processed_outputs.update(_processed_outputs) processed_data.update(passthrough_data)
return processed_outputs return processed_data
def _get_prompt_updates( def _get_prompt_updates(
self, self,
@@ -261,34 +261,44 @@ class HCXVisionV2MultiModalProcessor(
): ):
"""Multimodal processor for HyperCLOVAX V2 (32B Think model).""" """Multimodal processor for HyperCLOVAX V2 (32B Think model)."""
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
images = mm_data.get("images") images = mm_data.get("images")
videos = mm_data.get("videos") videos = mm_data.get("videos")
# Get the HF processor # Get the HF processor
hf_processor = self.info.get_hf_processor(**mm_kwargs) hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
# Build data dict for HF processor (images/videos only) # Build data dict for HF processor (images/videos only)
# NOTE: We pass the prompt as-is without token normalization. # NOTE: We pass the prompt as-is without token normalization.
# Token expansion is handled by vLLM via _get_prompt_updates. # Token expansion is handled by vLLM via _get_prompt_updates.
data: dict[str, object] = dict( data: dict[str, object] = dict(
text=prompt, text=prompt_text,
images=images, images=images,
videos=videos, videos=videos,
) )
processed_outputs = self.info.ctx.call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
hf_processor=hf_processor, hf_processor=hf_processor,
data=data, data=data,
kwargs=mm_kwargs, kwargs=hf_processor_mm_kwargs,
) )
processed_data.update(passthrough_data)
return processed_outputs return processed_data
def _get_prompt_updates( def _get_prompt_updates(
self, self,
+26 -19
View File
@@ -239,51 +239,58 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]): class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
# Text-only input not supported in composite processor valid_mm_items = mm_items.select(
if not (images := mm_data.get("images", [])): {k for k, c in mm_items.get_all_counts().items() if c > 0}
prompt_ids = self.info.get_tokenizer().encode(prompt) )
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
if not mm_data:
return BatchFeature(dict(passthrough_data))
image_processor = self.info.get_hf_processor().image_processor image_processor = self.info.get_hf_processor().image_processor
if getattr(image_processor, "backend", "pil") == "pil": if getattr(image_processor, "backend", "pil") == "pil":
mm_kwargs = {"input_data_format": "channels_last", **mm_kwargs} hf_processor_mm_kwargs = {
"input_data_format": "channels_last",
**hf_processor_mm_kwargs,
}
processed_outputs = super()._call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
prompt, self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data, mm_data,
mm_kwargs, hf_processor_mm_kwargs,
) )
images = mm_data.get("images", [])
mm_items = self.info.parse_mm_data({"image": images}, validate=False) mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems) parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [ image_sizes = [
parsed_images.get_image_size(i) for i in range(len(parsed_images)) parsed_images.get_image_size(i) for i in range(len(parsed_images))
] ]
hf_processor = self.info.get_hf_processor(**mm_kwargs) hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
num_patches = [ num_patches = [
self.info.get_num_patches( self.info.get_num_patches(
image_width=size.width, image_width=size.width,
image_height=size.height, image_height=size.height,
processor=hf_processor, processor=hf_processor,
mm_kwargs=mm_kwargs, mm_kwargs=hf_processor_mm_kwargs,
) )
for size in image_sizes for size in image_sizes
] ]
processed_outputs["num_patches"] = torch.tensor(num_patches) processed_data["num_patches"] = torch.tensor(num_patches)
# Remove the extra batch dimension # Remove the extra batch dimension
processed_outputs["pixel_values"].squeeze_(0) processed_data["pixel_values"].squeeze_(0)
processed_outputs["pixel_attention_mask"].squeeze_(0) processed_data["pixel_attention_mask"].squeeze_(0)
return processed_outputs processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+51 -25
View File
@@ -332,19 +332,28 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]): class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]):
"""Basic image-only MultiModalProcessor for InternS1-style models.""" """Basic image-only MultiModalProcessor for InternS1-style models."""
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data) mm_data = dict(mm_data)
videos = mm_data.pop("videos", []) videos = mm_data.pop("videos", [])
images = mm_data.pop("images", []) images = mm_data.pop("images", [])
assert isinstance(videos, list) assert isinstance(videos, list)
assert isinstance(images, list) assert isinstance(images, list)
hf_processor = self.info.get_hf_processor(**mm_kwargs) hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = hf_processor.tokenizer tokenizer = hf_processor.tokenizer
video_token_id = tokenizer.encode( video_token_id = tokenizer.encode(
hf_processor.video_token, add_special_tokens=False hf_processor.video_token, add_special_tokens=False
@@ -352,23 +361,29 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
assert len(video_token_id) == 1 assert len(video_token_id) == 1
video_token_id = video_token_id[0] video_token_id = video_token_id[0]
prompt = re.sub(hf_processor.image_token, "<image_placeholder>", prompt) prompt_text = re.sub(
prompt = re.sub(hf_processor.video_token, "<video_placeholder>", prompt) hf_processor.image_token, "<image_placeholder>", prompt_text
)
prompt_text = re.sub(
hf_processor.video_token, "<video_placeholder>", prompt_text
)
image_outputs = {} image_outputs = {}
if images: if images:
image_pixel_values = [] image_pixel_values = []
for image in images: for image in images:
processed_outputs = super()._call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
prompt=hf_processor.image_token, self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data={"images": image}, dict(text=hf_processor.image_token, **{"images": image}),
mm_kwargs=mm_kwargs, hf_processor_mm_kwargs,
) )
image_pixel_values.append(processed_outputs.pop("pixel_values")) image_pixel_values.append(processed_data.pop("pixel_values"))
input_ids = processed_outputs.pop("input_ids") input_ids = processed_data.pop("input_ids")
image_placeholder = tokenizer.batch_decode(input_ids)[0] image_placeholder = tokenizer.batch_decode(input_ids)[0]
prompt = prompt.replace("<image_placeholder>", image_placeholder, 1) prompt_text = prompt_text.replace(
"<image_placeholder>", image_placeholder, 1
)
num_patches = [len(item) for item in image_pixel_values] num_patches = [len(item) for item in image_pixel_values]
image_outputs = { image_outputs = {
@@ -381,18 +396,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
if videos: if videos:
video_pixel_values = [] video_pixel_values = []
for video in videos: for video in videos:
processed_outputs = super()._call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
prompt=hf_processor.video_token, self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data={"videos": video}, dict(text=hf_processor.video_token, **{"videos": video}),
mm_kwargs=mm_kwargs, hf_processor_mm_kwargs,
) )
video_pixel_values.append(processed_outputs.pop("pixel_values")) video_pixel_values.append(processed_data.pop("pixel_values"))
input_ids = processed_outputs.pop("input_ids") input_ids = processed_data.pop("input_ids")
input_ids[input_ids == hf_processor.image_token_id] = video_token_id input_ids[input_ids == hf_processor.image_token_id] = video_token_id
video_placeholder = tokenizer.batch_decode(input_ids)[0] video_placeholder = tokenizer.batch_decode(input_ids)[0]
prompt = prompt.replace("<video_placeholder>", video_placeholder, 1) prompt_text = prompt_text.replace(
"<video_placeholder>", video_placeholder, 1
)
num_frames = [len(item) for item in video_pixel_values] num_frames = [len(item) for item in video_pixel_values]
video_outputs = { video_outputs = {
@@ -401,11 +418,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
"video_token_id": torch.tensor(video_token_id), "video_token_id": torch.tensor(video_token_id),
} }
prompt = re.sub("<image_placeholder>", hf_processor.image_token, prompt) prompt_text = re.sub(
prompt = re.sub("<video_placeholder>", hf_processor.video_token, prompt) "<image_placeholder>", hf_processor.image_token, prompt_text
text_outputs = tokenizer(prompt, return_tensors="pt") )
prompt_text = re.sub(
"<video_placeholder>", hf_processor.video_token, prompt_text
)
text_outputs = tokenizer(prompt_text, return_tensors="pt")
return BatchFeature({**text_outputs, **image_outputs, **video_outputs}) processed_data.update(text_outputs)
processed_data.update(image_outputs)
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+17 -19
View File
@@ -220,27 +220,21 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]): class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
"""Basic image-only MultiModalProcessor for InternVL-style models.""" """Basic image-only MultiModalProcessor for InternVL-style models."""
def _call_hf_processor( def _postprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature: ) -> BatchFeature:
processed_outputs = super()._call_hf_processor( hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
image_token_id = hf_processor.ctx_image_token_id image_token_id = hf_processor.ctx_image_token_id
# Since there may be extra tokens in the feature placeholders, # Since there may be extra tokens in the feature placeholders,
# we need to pass the image token ID to the model to select the # we need to pass the image token ID to the model to select the
# tokens to merge from the vision encoder outputs # tokens to merge from the vision encoder outputs
processed_outputs["image_token_id"] = torch.tensor(image_token_id) processed_data["image_token_id"] = torch.tensor(image_token_id)
return processed_outputs return processed_data
def _get_image_fields_config(self, hf_inputs: BatchFeature): def _get_image_fields_config(self, hf_inputs: BatchFeature):
image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0)) image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
@@ -455,19 +449,23 @@ class InternVLMultiModalProcessor(
): ):
"""InternVL MultiModalProcessor extended for video support""" """InternVL MultiModalProcessor extended for video support"""
def _call_hf_processor( def _postprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature: ) -> BatchFeature:
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs) processed_data = super()._postprocess_hf_mm_data(
mm_data,
hf_processor_mm_kwargs,
processed_data,
)
hf_processor = self.info.get_hf_processor(**mm_kwargs) hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
if (video_token_id := hf_processor.ctx_video_token_id) is not None: if (video_token_id := hf_processor.ctx_video_token_id) is not None:
processed_outputs["video_token_id"] = torch.tensor(video_token_id) processed_data["video_token_id"] = torch.tensor(video_token_id)
return processed_outputs return processed_data
def _get_video_fields_config(self, hf_inputs: BatchFeature): def _get_video_fields_config(self, hf_inputs: BatchFeature):
video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0)) video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
+19 -9
View File
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.pooler import DispatchPooler from vllm.model_executor.layers.pooler import DispatchPooler
from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import MultiModalDataItems
from vllm.sequence import IntermediateTensors from vllm.sequence import IntermediateTensors
from .interfaces import SupportsCrossEncoding, SupportsMultiModal, SupportsScoreTemplate from .interfaces import SupportsCrossEncoding, SupportsMultiModal, SupportsScoreTemplate
@@ -54,19 +55,28 @@ class JinaVLScorer(nn.Module):
class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor): class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
# NOTE: We should reverse the order of the mm_data because the valid_mm_items = mm_items.select(
# query prompt is placed after the document prompt in the score {k for k, c in mm_items.get_all_counts().items() if c > 0}
# template for JinaVLForRanking model, but in mm_data they are )
# stored in the opposite order (query first, then document). mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
for _, value in mm_data.items(): for _, value in mm_data.items():
value.reverse() value.reverse()
return super()._call_hf_processor(prompt, mm_data, mm_kwargs) processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
@MULTIMODAL_REGISTRY.register_processor( @MULTIMODAL_REGISTRY.register_processor(
+15 -9
View File
@@ -461,17 +461,21 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
def media_token_id(self) -> int: def media_token_id(self) -> int:
return self.info.get_hf_config().text_config.eos_token_id + 1 return self.info.get_hf_config().text_config.eos_token_id + 1
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
"""Run the underlying HF processor on text and image data.""" """Run the underlying HF processor on text and image data."""
# Text-only input is handled as a special case here. valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data or not mm_data.get("images", []): if not mm_data or not mm_data.get("images", []):
prompt_ids = self.info.get_tokenizer().encode(prompt) return BatchFeature(dict(passthrough_data))
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
# Images # Images
image_inputs = mm_data.get("images", []) image_inputs = mm_data.get("images", [])
@@ -493,7 +497,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
media_token = tokenizer.convert_ids_to_tokens([self.media_token_id])[0] media_token = tokenizer.convert_ids_to_tokens([self.media_token_id])[0]
prompt_replaced = prompt.replace("<image>", media_token) prompt_replaced = prompt_text.replace("<image>", media_token)
input_ids = tokenizer.encode(prompt_replaced) input_ids = tokenizer.encode(prompt_replaced)
input_ids = torch.tensor(input_ids) input_ids = torch.tensor(input_ids)
@@ -527,7 +531,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
image_token_thw=torch.tensor(image_meta["image_token_thw"]), image_token_thw=torch.tensor(image_meta["image_token_thw"]),
pixel_sizes=torch.tensor(pixel_sizes), pixel_sizes=torch.tensor(pixel_sizes),
) )
return BatchFeature(combined_outputs, tensor_type="pt") processed_data = BatchFeature(combined_outputs, tensor_type="pt")
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates( def _get_prompt_updates(
self, self,
+2 -9
View File
@@ -1162,15 +1162,8 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):
class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]): class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, return self.dummy_inputs.get_dummy_text(mm_counts)
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_keye.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates( def _get_prompt_updates(
self, self,
+2 -9
View File
@@ -378,15 +378,8 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]): class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, return self.dummy_inputs.get_dummy_text(mm_counts)
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_keye.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates( def _get_prompt_updates(
self, self,
+18 -8
View File
@@ -37,6 +37,7 @@ from vllm.multimodal.parse import (
DictEmbeddingItems, DictEmbeddingItems,
ModalityData, ModalityData,
ModalityDataItems, ModalityDataItems,
MultiModalDataItems,
MultiModalDataParser, MultiModalDataParser,
) )
from vllm.multimodal.processing import ( from vllm.multimodal.processing import (
@@ -226,13 +227,20 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser):
class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]): class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]):
"""vLLM multi-modal processor wrapper for Kimi-Audio.""" """vLLM multi-modal processor wrapper for Kimi-Audio."""
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
"""Call the HuggingFace processor.""" """Call the HuggingFace processor."""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
# Convert mm_data format: {'audios': [...]} -> {'audio': ...} # Convert mm_data format: {'audios': [...]} -> {'audio': ...}
mm_data = dict(mm_data) mm_data = dict(mm_data)
audios = mm_data.pop("audios", []) audios = mm_data.pop("audios", [])
@@ -252,11 +260,13 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
mm_data["audio"] = audio_arrays mm_data["audio"] = audio_arrays
# Use the context's call_hf_processor for proper handling # Use the context's call_hf_processor for proper handling
return self.info.ctx.call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs), self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt, **mm_data), mm_data,
mm_kwargs, hf_processor_mm_kwargs,
) )
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
-10
View File
@@ -258,16 +258,6 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
grid_thws=MultiModalFieldConfig.batched("vision_chunk", keep_on_cpu=True), grid_thws=MultiModalFieldConfig.batched("vision_chunk", keep_on_cpu=True),
) )
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path because vision chunk
# is not considered
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates( def _get_prompt_updates(
self, self,
mm_items: MultiModalDataItems, mm_items: MultiModalDataItems,
+3
View File
@@ -237,6 +237,9 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]): class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
hf_inputs: BatchFeature, hf_inputs: BatchFeature,
+13 -20
View File
@@ -393,45 +393,38 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):
class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]): class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature: ) -> BatchFeature:
# Text-only input not supported in composite processor if not mm_data:
if not (images := mm_data.get("images", [])): return processed_data
prompt_ids = self.info.get_tokenizer().encode(
prompt, add_special_tokens=False
)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
processed_outputs = super()._call_hf_processor(
prompt,
mm_data,
mm_kwargs,
)
images = mm_data.get("images", [])
mm_items = self.info.parse_mm_data({"image": images}, validate=False) mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems) parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [ image_sizes = [
parsed_images.get_image_size(i) for i in range(len(parsed_images)) parsed_images.get_image_size(i) for i in range(len(parsed_images))
] ]
hf_processor = self.info.get_hf_processor(**mm_kwargs) hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
num_patches = [ num_patches = [
self.info.get_num_patches( self.info.get_num_patches(
image_width=size.width, image_width=size.width,
image_height=size.height, image_height=size.height,
processor=hf_processor, processor=hf_processor,
mm_kwargs=mm_kwargs, mm_kwargs=hf_processor_mm_kwargs,
) )
for size in image_sizes for size in image_sizes
] ]
processed_outputs["num_patches"] = torch.tensor(num_patches) processed_data["num_patches"] = torch.tensor(num_patches)
return processed_outputs return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+21 -22
View File
@@ -39,20 +39,19 @@ _I = TypeVar("_I", bound=Mistral3ProcessingInfo)
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]): class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, return self.dummy_inputs.get_dummy_text(mm_counts)
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
# NOTE: LightOnOCR does not use break/end tokens, so we remove them here. def _postprocess_hf_mm_data(
input_ids = processed_outputs.get("input_ids") self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
input_ids = processed_data.get("input_ids")
if input_ids is not None: if input_ids is not None:
processor = self.info.get_hf_processor() processor = self.info.get_hf_processor()
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
@@ -67,22 +66,22 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
torch.tensor([break_id, end_id]), torch.tensor([break_id, end_id]),
) )
processed_outputs["input_ids"] = input_ids[keep_mask].unsqueeze(0) processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0)
if "attention_mask" in processed_outputs: if "attention_mask" in processed_data:
processed_outputs["attention_mask"] = processed_outputs[ processed_data["attention_mask"] = processed_data["attention_mask"][
"attention_mask" keep_mask
][keep_mask].unsqueeze(0) ].unsqueeze(0)
# un-pad pixel_values per-image so caches remain independent. # un-pad pixel_values per-image so caches remain independent.
pixel_values = processed_outputs.get("pixel_values") pixel_values = processed_data.get("pixel_values")
if pixel_values is not None: if pixel_values is not None:
image_sizes = processed_outputs["image_sizes"] image_sizes = processed_data["image_sizes"]
assert len(pixel_values) == len(image_sizes) assert len(pixel_values) == len(image_sizes)
processed_outputs["pixel_values"] = [ processed_data["pixel_values"] = [
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes) p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
] ]
return processed_outputs return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+15 -15
View File
@@ -321,30 +321,30 @@ class PixtralHFProcessingInfo(BaseLlavaProcessingInfo):
class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]): class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, return self.dummy_inputs.get_dummy_text(mm_counts)
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
pixel_values = processed_outputs.get("pixel_values") def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
pixel_values = processed_data.get("pixel_values")
if pixel_values is not None: if pixel_values is not None:
# Avoid padding since we need the output for each image to be # Avoid padding since we need the output for each image to be
# independent of other images for the cache to work correctly # independent of other images for the cache to work correctly
image_sizes = processed_outputs["image_sizes"] image_sizes = processed_data["image_sizes"]
assert len(pixel_values) == len(image_sizes) assert len(pixel_values) == len(image_sizes)
processed_outputs["pixel_values"] = [ processed_data["pixel_values"] = [
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes) p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
] ]
return processed_outputs return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+3
View File
@@ -211,6 +211,9 @@ class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]):
class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]): class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
hf_inputs: BatchFeature, hf_inputs: BatchFeature,
+31 -24
View File
@@ -321,22 +321,33 @@ class LlavaOnevisionMultiModalProcessor(
pixel_values_videos=MultiModalFieldConfig.batched("video"), pixel_values_videos=MultiModalFieldConfig.batched("video"),
) )
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data) mm_data = dict(mm_data)
videos = mm_data.pop("videos", []) videos = mm_data.pop("videos", [])
assert isinstance(videos, list) assert isinstance(videos, list)
if not videos: if not videos:
return super()._call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
prompt=prompt, self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data=mm_data, dict(text=prompt_text, **mm_data),
mm_kwargs=mm_kwargs, hf_processor_mm_kwargs,
) )
processed_data.update(passthrough_data)
return processed_data
# LLaVA-OneVision processor doesn't support multiple videos # LLaVA-OneVision processor doesn't support multiple videos
# with different sizes when converting back to tensors # with different sizes when converting back to tensors
@@ -346,19 +357,13 @@ class LlavaOnevisionMultiModalProcessor(
image_token = processor.image_token image_token = processor.image_token
video_token = processor.video_token video_token = processor.video_token
text_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data={},
mm_kwargs=mm_kwargs,
)
images = mm_data.pop("images", []) images = mm_data.pop("images", [])
assert isinstance(images, list) assert isinstance(images, list)
if images: if images:
processor_outputs = super()._call_hf_processor( processor_outputs = self.info.ctx.call_hf_processor(
prompt=image_token * len(images), self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data={"images": images}, dict(text=image_token * len(images), **{"images": images}),
mm_kwargs=mm_kwargs, hf_processor_mm_kwargs,
) )
image_outputs = { image_outputs = {
k: v k: v
@@ -370,10 +375,10 @@ class LlavaOnevisionMultiModalProcessor(
pixel_values_videos = [] pixel_values_videos = []
for video in videos: for video in videos:
item_outputs = super()._call_hf_processor( item_outputs = self.info.ctx.call_hf_processor(
prompt=video_token, self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data={"videos": video}, dict(text=video_token, **{"videos": video}),
mm_kwargs=mm_kwargs, hf_processor_mm_kwargs,
) )
pixel_values_videos.append(item_outputs["pixel_values_videos"][0]) pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
@@ -381,11 +386,13 @@ class LlavaOnevisionMultiModalProcessor(
video_outputs = {"pixel_values_videos": pixel_values_videos} video_outputs = {"pixel_values_videos": pixel_values_videos}
combined_outputs = dict( combined_outputs = dict(
text_outputs, input_ids=prompt_text,
**image_outputs, **image_outputs,
**video_outputs, **video_outputs,
) )
return BatchFeature(combined_outputs) processed_data = BatchFeature(combined_outputs)
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates( def _get_prompt_updates(
self, self,
+48 -33
View File
@@ -286,7 +286,7 @@ def _validate_video_sources(paths, model_config) -> list[str]:
# participates in the standard decode-stage pipeline. # participates in the standard decode-stage pipeline.
# #
# * codec backend: NOT a loader. OV2's codec path needs the video path string # * codec backend: NOT a loader. OV2's codec path needs the video path string
# to survive into ``_call_hf_processor``, where the HF processor builds the # to survive into ``_apply_hf_processor_main``, where the HF processor builds the
# codec canvas + smart_resize + patchify # codec canvas + smart_resize + patchify
# (pixel_values/image_grid_thw/patch_positions). That transform is # (pixel_values/image_grid_thw/patch_positions). That transform is
# path-level and inseparable; it cannot be reconstructed from pre-decoded # path-level and inseparable; it cannot be reconstructed from pre-decoded
@@ -302,7 +302,7 @@ def prepare_codec_video_input(video_path: str) -> tuple:
Returns ``(dummy_ndarray, metadata)`` where the ndarray satisfies the Returns ``(dummy_ndarray, metadata)`` where the ndarray satisfies the
parser's 4-D shape check and the metadata carries the actual path to parser's 4-D shape check and the metadata carries the actual path to
our ``_call_hf_processor``. Use as:: our ``_apply_hf_processor_main``. Use as::
multi_modal_data = {"video": prepare_codec_video_input("foo.mp4")} multi_modal_data = {"video": prepare_codec_video_input("foo.mp4")}
@@ -1283,7 +1283,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
# ``video_needs_metadata=True`` makes the parser preserve both the # ``video_needs_metadata=True`` makes the parser preserve both the
# ``(frames, metadata)`` tuples from the frame backend and the # ``(frames, metadata)`` tuples from the frame backend and the
# ``(dummy, {marker: path})`` tuples from prepare_codec_video_input; # ``(dummy, {marker: path})`` tuples from prepare_codec_video_input;
# both are dispatched by metadata content in ``_call_hf_processor``. # both are dispatched by metadata content in ``_apply_hf_processor_main``.
return LlavaOnevision2MultiModalDataParser( return LlavaOnevision2MultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size, self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True, video_needs_metadata=True,
@@ -1547,18 +1547,23 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
class LlavaOnevision2MultiModalProcessor( class LlavaOnevision2MultiModalProcessor(
BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo] BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]
): ):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
# The wrapped OV2 processor is a bare custom class without the standard valid_mm_items = mm_items.select(
# ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path {k for k, c in mm_items.get_all_counts().items() if c > 0}
# fails; overriding this method routes the base class to call us )
# directly. mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs) if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
merged_kwargs.setdefault("return_tensors", "pt") merged_kwargs.setdefault("return_tensors", "pt")
call_kwargs = { call_kwargs = {
k: v k: v
@@ -1627,21 +1632,25 @@ class LlavaOnevision2MultiModalProcessor(
if len(codec_video_paths) > 1 if len(codec_video_paths) > 1
else codec_video_paths[0] else codec_video_paths[0]
) )
# Route through the base ``_call_hf_processor`` so float-tensor # Route through ``ctx.call_hf_processor`` so float-tensor
# dtype postprocessing is applied automatically; inject # dtype postprocessing is applied automatically; inject
# ``video_backend="codec"`` via mm_kwargs so the wrapped processor # ``video_backend="codec"`` via mm_kwargs so the wrapped processor
# dispatches to its codec branch. # dispatches to its codec branch.
output = super()._call_hf_processor( output = self.info.ctx.call_hf_processor(
prompt=prompt, self.info.get_hf_processor(
mm_data=mm_data, **{**hf_processor_mm_kwargs, "video_backend": "codec"}
mm_kwargs={**mm_kwargs, "video_backend": "codec"}, ),
dict(text=prompt_text, **mm_data),
{**hf_processor_mm_kwargs, "video_backend": "codec"},
) )
data = dict(output) data = dict(output)
return BatchFeature( processed_data = BatchFeature(
self._rename_codec_outputs_to_video( self._rename_codec_outputs_to_video(
data, codec_video_paths, hf_processor data, codec_video_paths, hf_processor
) )
) )
processed_data.update(passthrough_data)
return processed_data
# ---- Frame backend (registered LlavaOnevision2VideoBackend) ------ # ---- Frame backend (registered LlavaOnevision2VideoBackend) ------
# Every non-codec video reaches here as a ``(frames_ndarray, metadata)`` # Every non-codec video reaches here as a ``(frames_ndarray, metadata)``
@@ -1657,7 +1666,9 @@ class LlavaOnevision2MultiModalProcessor(
# local-file gating is enforced by the connector before decoding. # local-file gating is enforced by the connector before decoding.
if videos_present: if videos_present:
timestamp_decimals = int( timestamp_decimals = int(
mm_kwargs.get("timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS) hf_processor_mm_kwargs.get(
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
)
) )
per_video_frames: list[list[Image.Image]] = [] per_video_frames: list[list[Image.Image]] = []
@@ -1671,7 +1682,7 @@ class LlavaOnevision2MultiModalProcessor(
# ``<{t} seconds><|vision_start|><|image_pad|><|vision_end|>`` # ``<{t} seconds><|vision_start|><|image_pad|><|vision_end|>``
# blocks (matches the OV2 hf-chat reference exactly). # blocks (matches the OV2 hf-chat reference exactly).
new_prompt = _expand_video_markers_in_prompt( new_prompt = _expand_video_markers_in_prompt(
prompt, prompt_text,
per_video_timestamps, per_video_timestamps,
timestamp_decimals=timestamp_decimals, timestamp_decimals=timestamp_decimals,
) )
@@ -1700,7 +1711,7 @@ class LlavaOnevision2MultiModalProcessor(
row_is_video: list[bool] = [] row_is_video: list[bool] = []
vid_idx = 0 vid_idx = 0
img_idx = 0 img_idx = 0
for marker in marker_pattern.finditer(prompt): for marker in marker_pattern.finditer(prompt_text):
if marker.lastgroup == "video": if marker.lastgroup == "video":
frames = per_video_frames[vid_idx] frames = per_video_frames[vid_idx]
vid_idx += 1 vid_idx += 1
@@ -1723,15 +1734,15 @@ class LlavaOnevision2MultiModalProcessor(
merged_mm_data.pop("videos", None) merged_mm_data.pop("videos", None)
merged_mm_data["images"] = flat_frames merged_mm_data["images"] = flat_frames
# Route through the base ``_call_hf_processor`` (applies float-tensor # Route through ``ctx.call_hf_processor`` (applies float-tensor
# dtype postprocessing automatically). The wrapped processor's image # dtype postprocessing automatically). The wrapped processor's image
# branch ignores video/codec-only kwargs and does not forward extra # branch ignores video/codec-only kwargs and does not forward extra
# **kwargs to the image processor, so passing the full merged kwarg # **kwargs to the image processor, so passing the full merged kwarg
# set here is a no-op beyond return_tensors/padding. # set here is a no-op beyond return_tensors/padding.
output = super()._call_hf_processor( output = self.info.ctx.call_hf_processor(
prompt=new_prompt, self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data=merged_mm_data, dict(text=new_prompt, **merged_mm_data),
mm_kwargs=mm_kwargs, hf_processor_mm_kwargs,
) )
data = dict(output) data = dict(output)
@@ -1788,17 +1799,21 @@ class LlavaOnevision2MultiModalProcessor(
(len(per_video_timestamps),), dtype=torch.long (len(per_video_timestamps),), dtype=torch.long
) )
return BatchFeature(data) processed_data = BatchFeature(data)
processed_data.update(passthrough_data)
return processed_data
# ---- Image-only / text-only call -------------------------------- # ---- Image-only / text-only call --------------------------------
# No videos present: delegate to the base ``_call_hf_processor``, which # No videos present: delegate to ``ctx.call_hf_processor``, which
# runs the wrapped processor over the (possibly empty) image set and # runs the wrapped processor over the (possibly empty) image set and
# applies float-tensor dtype postprocessing automatically. # applies float-tensor dtype postprocessing automatically.
return super()._call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
prompt=prompt, self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data=mm_data, dict(text=prompt_text, **mm_data),
mm_kwargs=mm_kwargs, hf_processor_mm_kwargs,
) )
processed_data.update(passthrough_data)
return processed_data
def _rename_codec_outputs_to_video( def _rename_codec_outputs_to_video(
self, self,
+9 -20
View File
@@ -26,7 +26,7 @@
import collections import collections
import collections.abc import collections.abc
from collections.abc import Callable, Iterable, Mapping, Sequence from collections.abc import Callable, Iterable, Mapping, Sequence
from typing import Annotated, Any, TypeAlias, cast from typing import Annotated, TypeAlias, cast
import numpy as np import numpy as np
import torch import torch
@@ -583,15 +583,17 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
class MiDashengLMMultiModalProcessor( class MiDashengLMMultiModalProcessor(
BaseMultiModalProcessor[MiDashengLMProcessingInfo] BaseMultiModalProcessor[MiDashengLMProcessingInfo]
): ):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any], hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", []) audios = mm_data.pop("audios", [])
# + Padding
min_audio_len = self.info.get_min_audio_len() min_audio_len = self.info.get_min_audio_len()
processed_audios = [ processed_audios = [
np.pad( np.pad(
@@ -608,20 +610,7 @@ class MiDashengLMMultiModalProcessor(
if processed_audios: if processed_audios:
mm_data["audio"] = processed_audios mm_data["audio"] = processed_audios
if not mm_data.get("audio", []): return mm_data, hf_processor_mm_kwargs
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
mm_kwargs = dict(
**mm_kwargs,
)
return super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+18 -5
View File
@@ -912,16 +912,23 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio") fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")
return fields return fields
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo. """Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.
Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses
the singular form. the singular form.
""" """
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
# Remap audios → audio (MiMoOmniProcessor uses singular param name) # Remap audios → audio (MiMoOmniProcessor uses singular param name)
if "audios" in mm_data: if "audios" in mm_data:
mm_data = {**mm_data, "audio": mm_data["audios"]} mm_data = {**mm_data, "audio": mm_data["audios"]}
@@ -999,7 +1006,13 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
mm_data = {**mm_data, "videos": converted} mm_data = {**mm_data, "videos": converted}
return super()._call_hf_processor(prompt, mm_data, mm_kwargs) processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates( def _get_prompt_updates(
self, self,
+1 -1
View File
@@ -448,7 +448,7 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems): if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems):
audio_inputs = {} audio_inputs = {}
else: else:
audio_inputs = self._base_call_hf_processor( audio_inputs = self._call_hf_processor_on_prompts(
prompts=[self.info.audio_pattern] * len(parsed_audios), prompts=[self.info.audio_pattern] * len(parsed_audios),
mm_data={"audios": [[audio] for audio in parsed_audios]}, mm_data={"audios": [[audio] for audio in parsed_audios]},
mm_kwargs={**mm_kwargs, "chunk_input": True}, mm_kwargs={**mm_kwargs, "chunk_input": True},
+26 -18
View File
@@ -881,7 +881,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
if isinstance(parsed_images, MiniCPMVImageEmbeddingItems): if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
image_inputs = {} image_inputs = {}
else: else:
image_inputs = self._base_call_hf_processor( image_inputs = self._call_hf_processor_on_prompts(
prompts=[self.info.image_pattern] * len(parsed_images), prompts=[self.info.image_pattern] * len(parsed_images),
mm_data={"images": [[image] for image in parsed_images]}, mm_data={"images": [[image] for image in parsed_images]},
mm_kwargs=mm_kwargs, mm_kwargs=mm_kwargs,
@@ -906,7 +906,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems): if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
video_inputs = {} video_inputs = {}
else: else:
video_inputs = self._base_call_hf_processor( video_inputs = self._call_hf_processor_on_prompts(
prompts=[ prompts=[
self.info.image_pattern * len(video) for video in parsed_videos self.info.image_pattern * len(video) for video in parsed_videos
], ],
@@ -983,7 +983,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
return new_token_ids, placeholders return new_token_ids, placeholders
def _base_call_hf_processor( def _call_hf_processor_on_prompts(
self, self,
prompts: list[str], prompts: list[str],
mm_data: Mapping[str, Sequence[object]], mm_data: Mapping[str, Sequence[object]],
@@ -993,19 +993,19 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
) -> dict[str, NestedTensors]: ) -> dict[str, NestedTensors]:
# This processor supports zipping prompt and mm_data together # This processor supports zipping prompt and mm_data together
if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}: if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}:
inputs = super()._call_hf_processor( inputs = self.info.ctx.call_hf_processor(
prompt=prompts, # type: ignore self.info.get_hf_processor(**mm_kwargs),
mm_data=mm_data, dict(text=prompts, **mm_data),
mm_kwargs=mm_kwargs, mm_kwargs,
) )
else: else:
inputs = defaultdict[str, list[torch.Tensor]](list) inputs = defaultdict[str, list[torch.Tensor]](list)
for i, prompt in enumerate(prompts): for i, prompt in enumerate(prompts):
inputs_one = super()._call_hf_processor( inputs_one = self.info.ctx.call_hf_processor(
prompt=prompt, self.info.get_hf_processor(**mm_kwargs),
mm_data={k: v[i] for k, v in mm_data.items()}, dict(text=prompt, **{k: v[i] for k, v in mm_data.items()}),
mm_kwargs=mm_kwargs, mm_kwargs,
) )
for k, v in inputs_one.items(): for k, v in inputs_one.items():
@@ -1014,23 +1014,31 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
return {k: inputs[k] for k in out_keys} return {k: inputs[k] for k in out_keys}
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
input_ids = torch.tensor([tokenizer.encode(prompt)]) input_ids = torch.tensor([tokenizer.encode(prompt_text)])
mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs) mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
return BatchFeature( processed_data = BatchFeature(
{ {
"input_ids": input_ids, "input_ids": input_ids,
**mm_inputs, **mm_inputs,
} }
) )
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates( def _get_prompt_updates(
self, self,
+15 -15
View File
@@ -233,30 +233,30 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo])
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]): class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, return self.dummy_inputs.get_dummy_text(mm_counts)
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
pixel_values = processed_outputs.get("pixel_values") def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
pixel_values = processed_data.get("pixel_values")
if pixel_values is not None: if pixel_values is not None:
# Avoid padding since we need the output for each image to be # Avoid padding since we need the output for each image to be
# independent of other images for the cache to work correctly # independent of other images for the cache to work correctly
image_sizes = processed_outputs["image_sizes"] image_sizes = processed_data["image_sizes"]
assert len(pixel_values) == len(image_sizes) assert len(pixel_values) == len(image_sizes)
processed_outputs["pixel_values"] = [ processed_data["pixel_values"] = [
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes) p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
] ]
return processed_outputs return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+16 -16
View File
@@ -589,23 +589,23 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]): class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, return self.dummy_inputs.get_dummy_text(mm_counts)
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
processor = self.info.get_hf_processor(**mm_kwargs) def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_processor = processor.image_processor image_processor = processor.image_processor
vision_config = self.info.get_hf_config().vision_config vision_config = self.info.get_hf_config().vision_config
if processed_outputs.get("pixel_values") is not None: if processed_data.get("pixel_values") is not None:
assert "images" in mm_data, ( assert "images" in mm_data, (
"images expected to be in mm_data when pixel_values is present" "images expected to be in mm_data when pixel_values is present"
) )
@@ -636,10 +636,10 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
1 if r_h * r_w == 1 else 1 + r_h * r_w for (r_h, r_w) in aspect_ratios 1 if r_h * r_w == 1 else 1 + r_h * r_w for (r_h, r_w) in aspect_ratios
] ]
processed_outputs["aspect_ratios"] = torch.tensor(aspect_ratios) processed_data["aspect_ratios"] = torch.tensor(aspect_ratios)
processed_outputs["patches_per_image"] = torch.tensor(patches_per_image) processed_data["patches_per_image"] = torch.tensor(patches_per_image)
return processed_outputs return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+42 -39
View File
@@ -1123,17 +1123,44 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]): class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
def _call_hf_processor( def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
processor = self.info.get_hf_processor()
# The chat template is already applied to the prompt tokens
# Use message_format="none" to avoid applying it again
# Prepend an empty space if `always_start_with_space` is True
tokens = processor.get_tokens_input(
self.info.get_tokenizer().decode(prompt),
message_format="none",
always_start_with_space=True,
)
# Prepend a BOS token id to the tokens
return self.info.ctx.call_hf_processor(
processor.process,
dict(tokens=tokens),
)["input_ids"].tolist()
def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
hf_processor = self.info.get_hf_processor(**mm_kwargs) mm_counts = mm_items.get_all_counts()
processed_outputs = self.info.ctx.call_hf_processor(
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not processor_data:
return BatchFeature(dict(passthrough_data))
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
processed_data = self.info.ctx.call_hf_processor(
hf_processor.process, hf_processor.process,
dict(text=prompt, **mm_data), dict(text=prompt_text, **processor_data),
mm_kwargs, hf_processor_mm_kwargs,
) )
tokenizer = hf_processor.tokenizer tokenizer = hf_processor.tokenizer
@@ -1141,17 +1168,16 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
image_processor = hf_processor.image_processor image_processor = hf_processor.image_processor
input_ids: torch.Tensor = processed_outputs.pop("input_ids") processed_data.pop("input_ids")
processed_outputs["input_ids"] = input_ids.unsqueeze(0)
if (images := mm_data.get("images")) is not None: if (images := processor_data.get("images")) is not None:
mm_items = self.info.parse_mm_data({"image": images}, validate=False) mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems) parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [ image_sizes = [
parsed_images.get_image_size(i) for i in range(len(parsed_images)) parsed_images.get_image_size(i) for i in range(len(parsed_images))
] ]
feat_is_patch = processed_outputs["image_input_idx"] >= 0 feat_is_patch = processed_data["image_input_idx"] >= 0
tilings = [ tilings = [
self.info.select_tiling( self.info.select_tiling(
@@ -1165,35 +1191,12 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
num_crops = torch.tensor(tilings).prod(-1) + 1 num_crops = torch.tensor(tilings).prod(-1) + 1
assert num_crops.sum() == len(feat_is_patch) assert num_crops.sum() == len(feat_is_patch)
processed_outputs["num_crops"] = num_crops processed_data["num_crops"] = num_crops
processed_outputs["img_patch_id"] = image_patch_id processed_data["img_patch_id"] = image_patch_id
return processed_outputs processed_data.update(passthrough_data)
def _apply_hf_processor_tokens_only( return processed_data
self,
prompt_tokens: list[int],
) -> list[int]:
processor = self.info.get_hf_processor()
# The chat template is already applied to the prompt tokens
# Use message_format="none" to avoid applying it again
# Prepend an empty space if `always_start_with_space` is True
tokens = processor.get_tokens_input(
self.info.get_tokenizer().decode(prompt_tokens),
message_format="none",
always_start_with_space=True,
)
# Prepend a BOS token id to the tokens
processed_data = self.info.ctx.call_hf_processor(
processor.process,
dict(tokens=tokens),
)
prompt_ids = processed_data.pop("input_ids").tolist()
print(prompt_ids, len(prompt_ids))
return prompt_ids
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+41 -32
View File
@@ -1929,30 +1929,33 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
def _apply_hf_processor_tokens_only( def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
self,
prompt_tokens: list[int],
) -> list[int]:
processor = self.info.get_hf_processor() processor = self.info.get_hf_processor()
tokenizer = processor.tokenizer tokenizer = processor.tokenizer
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
if len(prompt_tokens) == 0 or prompt_tokens[0] != bos_token_id: if len(prompt) == 0 or prompt[0] != bos_token_id:
# Prepend the bos token to the prompt tokens # Prepend the bos token to the prompt tokens
prompt_tokens = [bos_token_id] + prompt_tokens prompt = [bos_token_id] + prompt
return prompt_tokens return prompt
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
mm_data = dict(mm_data) mm_counts = mm_items.get_all_counts()
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
mm_data = dict(processor_data)
hf_config = self.info.get_hf_config() hf_config = self.info.get_hf_config()
hf_processor = self.info.get_hf_processor(**mm_kwargs) hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature: def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature:
res = hf_processor(text=text, images=images, videos=videos, **kwargs) res = hf_processor(text=text, images=images, videos=videos, **kwargs)
@@ -1984,14 +1987,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
# NOTE: metadata.frames_indices indicates # NOTE: metadata.frames_indices indicates
# the sampled frames indices of pre-sampled videos, which is # the sampled frames indices of pre-sampled videos, which is
# used to calculate the timestamps. Make sure that # used to calculate the timestamps. Make sure that
# do_sample_frames in mm_kwargs is false for presampled videos. # do_sample_frames in hf_processor_mm_kwargs is false for
# presampled videos.
# NOTE: a copy of mm_kwargs is created to update do_sample_frames, # NOTE: a copy of hf_processor_mm_kwargs is created to update
# otherwise mm_hash for the object will be incorrect. # do_sample_frames, otherwise mm_hash for the object will be
video_mm_kwargs = dict(**mm_kwargs) # incorrect.
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
if "do_sample_frames" not in video_mm_kwargs: if "do_sample_frames" not in video_mm_kwargs:
# molmo_utils already has "do_sample_frames" in # molmo_utils already has "do_sample_frames" in
# mm_kwargs, don't overwrite it. # hf_processor_mm_kwargs, don't overwrite it.
video_mm_kwargs["do_sample_frames"] = metadata.get( video_mm_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", False "do_sample_frames", False
) )
@@ -2015,7 +2020,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
input_ids = input_ids[:, 1:] input_ids = input_ids[:, 1:]
video_string = tokenizer.batch_decode(input_ids)[0] video_string = tokenizer.batch_decode(input_ids)[0]
prompt = prompt.replace(VIDEO_PROMPT, video_string, 1) prompt_text = prompt_text.replace(VIDEO_PROMPT, video_string, 1)
video_grids = video_outputs.pop("video_grids") video_grids = video_outputs.pop("video_grids")
assert video_grids[:, 0].sum() == len( assert video_grids[:, 0].sum() == len(
@@ -2054,10 +2059,10 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
else: else:
all_video_outputs = dict() all_video_outputs = dict()
processed_outputs = self.info.ctx.call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
patched_call, patched_call,
dict(text=prompt, **mm_data), dict(text=prompt_text, **mm_data),
mm_kwargs, hf_processor_mm_kwargs,
) )
if (images := mm_data.get("images")) is not None: if (images := mm_data.get("images")) is not None:
@@ -2077,22 +2082,22 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
for image_size in image_sizes for image_size in image_sizes
] ]
num_crops = torch.tensor(tilings).prod(-1) + 1 num_crops = torch.tensor(tilings).prod(-1) + 1
assert sum(num_crops) == len(processed_outputs["pixel_values"]) assert sum(num_crops) == len(processed_data["pixel_values"])
assert sum(num_crops) == processed_outputs["image_num_crops"].sum().item() assert sum(num_crops) == processed_data["image_num_crops"].sum().item()
image_grids = processed_outputs.pop("image_grids") image_grids = processed_data.pop("image_grids")
image_num_pooled_patches = image_grids[:, :2].prod(dim=1) + image_grids[ image_num_pooled_patches = image_grids[:, :2].prod(dim=1) + image_grids[
:, 2: :, 2:
].prod(dim=1) ].prod(dim=1)
processed_outputs["image_num_pooled_patches"] = image_num_pooled_patches processed_data["image_num_pooled_patches"] = image_num_pooled_patches
n_patches = processed_outputs["pixel_values"].shape[1] n_patches = processed_data["pixel_values"].shape[1]
processed_outputs["image_num_patches"] = ( processed_data["image_num_patches"] = (
processed_outputs["image_num_crops"] * n_patches processed_data["image_num_crops"] * n_patches
) )
( (
processed_outputs["image_tokens"], processed_data["image_tokens"],
processed_outputs["num_image_tokens"], processed_data["num_image_tokens"],
) = build_flat_image_bool_length( ) = build_flat_image_bool_length(
image_grids, image_grids,
hf_config, hf_config,
@@ -2101,7 +2106,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
use_single_crop_start_token=hf_processor.use_single_crop_start_token, use_single_crop_start_token=hf_processor.use_single_crop_start_token,
) )
return BatchFeature({**processed_outputs, **all_video_outputs}) processed_data.update(all_video_outputs)
processed_data.update(passthrough_data)
processed_data.pop("input_ids")
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
-10
View File
@@ -942,16 +942,6 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
image_placeholder: str = "<image>" image_placeholder: str = "<image>"
bos_image_placeholder: str = "<|endoftext|><image>" bos_image_placeholder: str = "<|endoftext|><image>"
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Moondream3's processor handles images directly rather than exposing a
# separate `image_processor`, so keep the cache path on text+MM calls.
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
@cached_property @cached_property
def bos_image_placeholder_tokens(self) -> list[int]: def bos_image_placeholder_tokens(self) -> list[int]:
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
+17 -8
View File
@@ -1335,23 +1335,32 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo
class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]): class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
mm_data = dict(mm_data) mm_data = dict(mm_data)
audios = mm_data.pop("audios", []) audios = mm_data.pop("audios", [])
if audios: if audios:
mm_data["audio"] = audios mm_data["audio"] = audios
mm_kwargs = dict(mm_kwargs) hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs) processor_kwargs = _filter_moss_audio_processor_config(hf_processor_mm_kwargs)
return self.info.ctx.call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**processor_kwargs), self.info.get_hf_processor(**processor_kwargs),
dict(text=prompt, **mm_data), mm_data,
{}, {},
) )
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
@@ -461,24 +461,29 @@ class MossTranscribeDiarizeDummyInputsBuilder(
class MossTranscribeDiarizeMultiModalProcessor( class MossTranscribeDiarizeMultiModalProcessor(
BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo] BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]
): ):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, return self.dummy_inputs.get_dummy_text(mm_counts)
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
audios = _get_audios_from_mm_data(mm_data)
if not audios:
input_ids = tokenizer.encode(prompt, add_special_tokens=False)
return BatchFeature({"input_ids": [input_ids]}, tensor_type="pt")
processed = self.info.ctx.call_hf_processor( def _preprocess_hf_mm_data(
self.info.get_hf_processor(**mm_kwargs), self,
dict(text=prompt, audio=audios), mm_data: Mapping[str, object],
mm_kwargs, hf_processor_mm_kwargs: Mapping[str, object],
) ) -> tuple[Mapping[str, object], Mapping[str, object]]:
return _add_vllm_audio_metadata(processed, len(audios)) audios = _get_audios_from_mm_data(mm_data)
return dict(audio=audios), hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
audios = _get_audios_from_mm_data(mm_data)
if audios:
processed_data = _add_vllm_audio_metadata(processed_data, len(audios))
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+15 -7
View File
@@ -262,13 +262,19 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
class MuseGlimmerMultiModalProcessor( class MuseGlimmerMultiModalProcessor(
BaseMultiModalProcessor[MuseGlimmerProcessingInfo] BaseMultiModalProcessor[MuseGlimmerProcessingInfo]
): ):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
processor = self.info.get_hf_processor(**mm_kwargs) valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = processor.tokenizer tokenizer = processor.tokenizer
config = self.info.get_hf_config() config = self.info.get_hf_config()
images = mm_data.get("images", ()) images = mm_data.get("images", ())
@@ -276,7 +282,7 @@ class MuseGlimmerMultiModalProcessor(
if not isinstance(images, Sequence) or not isinstance(videos, Sequence): if not isinstance(images, Sequence) or not isinstance(videos, Sequence):
raise TypeError("MuseGlimmer multi-modal data must be a sequence") raise TypeError("MuseGlimmer multi-modal data must be a sequence")
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False) prompt_ids = tokenizer.encode(prompt_text, add_special_tokens=False)
image_sentinel_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN) image_sentinel_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)
video_sentinel_id = tokenizer.convert_tokens_to_ids(VIDEO_TOKEN) video_sentinel_id = tokenizer.convert_tokens_to_ids(VIDEO_TOKEN)
if prompt_ids.count(image_sentinel_id) != len(images): if prompt_ids.count(image_sentinel_id) != len(images):
@@ -354,7 +360,9 @@ class MuseGlimmerMultiModalProcessor(
video_pixel_values=video_pixels, video_pixel_values=video_pixels,
video_feature_sizes=torch.tensor(video_sizes), video_feature_sizes=torch.tensor(video_sizes),
) )
return BatchFeature(data=data, tensor_type=None) processed_data = BatchFeature(data=data, tensor_type=None)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+8 -19
View File
@@ -361,18 +361,8 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
class NanoNemotronVLMultiModalProcessor( class NanoNemotronVLMultiModalProcessor(
BaseMultiModalProcessor[NanoNemotronVLProcessingInfo] BaseMultiModalProcessor[NanoNemotronVLProcessingInfo]
): ):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, return self.dummy_inputs.get_dummy_text(mm_counts)
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Bypass `call_hf_processor_mm_only` by no-op overriding`_call_hf_processor`,
so it chooses this path:
`type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor`
"""
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_image_fields_config(self, hf_inputs: BatchFeature): def _get_image_fields_config(self, hf_inputs: BatchFeature):
if self.info.is_dynamic_tiler: if self.info.is_dynamic_tiler:
@@ -406,7 +396,9 @@ class NanoNemotronVLMultiModalProcessor(
) )
def _get_audio_fields_config(self, hf_inputs: BatchFeature): def _get_audio_fields_config(self, hf_inputs: BatchFeature):
audio_num_clips = torch.as_tensor(hf_inputs["audio_num_clips"]) audio_num_clips = torch.as_tensor(
hf_inputs.get("audio_num_clips", torch.empty(0))
)
return dict( return dict(
input_audio_features=MultiModalFieldConfig.flat_from_sizes( input_audio_features=MultiModalFieldConfig.flat_from_sizes(
@@ -733,11 +725,8 @@ class NanoNemotronVLMultiModalProcessor(
# Bypass the cached path: the HF processor must receive the # Bypass the cached path: the HF processor must receive the
# prompt (with injected <so_embedding>) and the audio data # prompt (with injected <so_embedding>) and the audio data
# together so it can perform audio-token replacement natively. # together so it can perform audio-token replacement natively.
( mm_info = self._apply_hf_processor(inputs, timing_ctx)
prompt_ids, prompt_ids = self._postprocess_prompt(inputs.prompt)
mm_info,
is_update_applied,
) = self._apply_hf_processor(inputs, timing_ctx)
with timing_ctx.record("apply_prompt_updates"): with timing_ctx.record("apply_prompt_updates"):
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates( prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
@@ -745,7 +734,7 @@ class NanoNemotronVLMultiModalProcessor(
prompt_ids=prompt_ids, prompt_ids=prompt_ids,
mm_kwargs=mm_info.kwargs, mm_kwargs=mm_info.kwargs,
mm_prompt_updates=mm_info.prompt_updates, mm_prompt_updates=mm_info.prompt_updates,
is_update_applied=is_update_applied, is_update_applied=self.hf_processor_applies_updates,
) )
mm_placeholder_ranges = { mm_placeholder_ranges = {
@@ -411,22 +411,6 @@ class NemotronParseMultiModalProcessor(
) -> list[int]: ) -> list[int]:
return [0] return [0]
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
else:
hf_processor = self.info.get_hf_processor()
tokenizer = hf_processor.tokenizer
processed_outputs = tokenizer(
prompt, add_special_tokens=False, return_tensors="pt"
)
return processed_outputs
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
hf_inputs: BatchFeature, hf_inputs: BatchFeature,
+10 -21
View File
@@ -341,41 +341,30 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
# -300 is image_atom token, filter them out # -300 is image_atom token, filter them out
return [vte_vocab_size + x + 300 for x in image_indicators if x < -300] return [vte_vocab_size + x + 300 for x in image_indicators if x < -300]
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature: ) -> BatchFeature:
if not mm_data: if not mm_data:
# Avoid warning from HF logger for text-only input return processed_data
tokenizer = self.info.get_tokenizer()
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
hf_processor = self.info.get_hf_processor() hf_processor = self.info.get_hf_processor()
image_indicators = [ image_indicators = [
hf_processor.construct_image_indicators(grid) hf_processor.construct_image_indicators(grid)
for grid in processed_outputs["grids"] for grid in processed_data["grids"]
] ]
indicator_tokens = [ indicator_tokens = [
self.image_indicators_to_visual_tokens(indicator) self.image_indicators_to_visual_tokens(indicator)
for indicator in image_indicators for indicator in image_indicators
] ]
processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens) processed_data["indicator_tokens"] = torch.tensor(indicator_tokens)
return processed_outputs
def _apply_hf_processor_tokens_only( return processed_data
self,
prompt_tokens: list[int],
) -> list[int]:
return prompt_tokens
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+12 -22
View File
@@ -332,53 +332,43 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
if x >= INDICATOR_IDS[0] if x >= INDICATOR_IDS[0]
] ]
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature: ) -> BatchFeature:
if not mm_data: if not mm_data:
# Avoid warning from HF logger for text-only input return processed_data
tokenizer = self.info.get_tokenizer()
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
hf_processor = self.info.get_hf_processor() hf_processor = self.info.get_hf_processor()
if "videos" in mm_data: if "videos" in mm_data:
visual_indicators = [ visual_indicators = [
hf_processor.construct_visual_indicators((1, 1, 1), True) hf_processor.construct_visual_indicators((1, 1, 1), True)
for grid in processed_outputs["video_grids"] for grid in processed_data["video_grids"]
] ]
indicator_tokens = [ indicator_tokens = [
self.visual_indicators_to_visual_tokens(indicator) self.visual_indicators_to_visual_tokens(indicator)
for indicator in visual_indicators for indicator in visual_indicators
] ]
processed_outputs["video_indicator_tokens"] = torch.tensor(indicator_tokens) processed_data["video_indicator_tokens"] = torch.tensor(indicator_tokens)
if "images" in mm_data: if "images" in mm_data:
visual_indicators = [ visual_indicators = [
hf_processor.construct_visual_indicators((1, 1, 1), False) hf_processor.construct_visual_indicators((1, 1, 1), False)
for grid in processed_outputs["grids"] for grid in processed_data["grids"]
] ]
indicator_tokens = [ indicator_tokens = [
self.visual_indicators_to_visual_tokens(indicator) self.visual_indicators_to_visual_tokens(indicator)
for indicator in visual_indicators for indicator in visual_indicators
] ]
processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens) processed_data["indicator_tokens"] = torch.tensor(indicator_tokens)
return processed_outputs
def _apply_hf_processor_tokens_only( return processed_data
self,
prompt_tokens: list[int],
) -> list[int]:
return prompt_tokens
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+29 -24
View File
@@ -244,32 +244,37 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing
class PaddleOCRVLMultiModalProcessor( class PaddleOCRVLMultiModalProcessor(
BaseMultiModalProcessor[PaddleOCRVLProcessingInfo] BaseMultiModalProcessor[PaddleOCRVLProcessingInfo]
): ):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
if mm_data: valid_mm_items = mm_items.select(
final_mm_kwargs = dict(mm_kwargs or {}) {k for k, c in mm_items.get_all_counts().items() if c > 0}
final_mm_kwargs.setdefault("images_kwargs", {}) )
# vLLM use PIL.Image, always set channel_last mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
processed_outputs = self.info.ctx.call_hf_processor( if not mm_data:
self.info.get_hf_processor(**final_mm_kwargs), return BatchFeature(dict(passthrough_data))
dict(text=prompt, **mm_data),
mm_kwargs, prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
)
num_patches_per_image = processed_outputs["image_grid_thw"].prod(-1) final_mm_kwargs = dict(hf_processor_mm_kwargs or {})
processed_outputs["pixel_values"] = processed_outputs["pixel_values"].split( final_mm_kwargs.setdefault("images_kwargs", {})
num_patches_per_image.tolist() # vLLM use PIL.Image, always set channel_last
) final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
else: processed_data = self.info.ctx.call_hf_processor(
tokenizer = self.info.get_tokenizer() self.info.get_hf_processor(**final_mm_kwargs),
processed_outputs = tokenizer( dict(text=prompt_text, **mm_data),
prompt, add_special_tokens=True, return_tensors="pt" hf_processor_mm_kwargs,
) )
return processed_outputs num_patches_per_image = processed_data["image_grid_thw"].prod(-1)
processed_data["pixel_values"] = processed_data["pixel_values"].split(
num_patches_per_image.tolist()
)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
-17
View File
@@ -152,23 +152,6 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo
class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingInfo]): class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
if not mm_data:
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
return super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
hf_inputs: BatchFeature, hf_inputs: BatchFeature,
+13 -13
View File
@@ -396,19 +396,19 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]): class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, return self.dummy_inputs.get_dummy_text(mm_counts)
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
input_ids = processed_outputs["input_ids"] def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
input_ids = processed_data["input_ids"]
assert isinstance(input_ids, torch.Tensor) assert isinstance(input_ids, torch.Tensor)
# Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids, # Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
@@ -416,7 +416,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
# Therefore, we need to do an early replacement here # Therefore, we need to do an early replacement here
input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID) input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID)
return processed_outputs return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+25 -16
View File
@@ -868,43 +868,52 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]): class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
if not mm_data: valid_mm_items = mm_items.select(
prompt_ids = self.info.get_tokenizer().encode(prompt) {k for k, c in mm_items.get_all_counts().items() if c > 0}
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) )
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
sr = self.info.get_feature_extractor(**mm_kwargs).sampling_rate if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
sr = self.info.get_feature_extractor(**hf_processor_mm_kwargs).sampling_rate
if audio_data := mm_data.get("audios", []): if audio_data := mm_data.get("audios", []):
mm_data["audios"] = [(data, sr) for data in audio_data] mm_data["audios"] = [(data, sr) for data in audio_data]
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs) processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
hf_processor = self.info.get_hf_processor(**mm_kwargs) hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
num_img_tokens = [ num_img_tokens = [
self.info.get_num_image_tokens( self.info.get_num_image_tokens(
image_width=img_size[0], image_width=img_size[0],
image_height=img_size[1], image_height=img_size[1],
processor=hf_processor, processor=hf_processor,
) )
for img_size in processed_outputs["image_sizes"] for img_size in processed_data["image_sizes"]
] ]
processed_outputs["num_img_tokens"] = num_img_tokens processed_data["num_img_tokens"] = num_img_tokens
audio_features = processed_outputs["input_audio_embeds"] audio_features = processed_data["input_audio_embeds"]
feature_sizes = [ feature_sizes = [
self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data
] ]
processed_outputs["input_audio_embeds"] = [ processed_data["input_audio_embeds"] = [
audio_features[idx, :size] for idx, size in enumerate(feature_sizes) audio_features[idx, :size] for idx, size in enumerate(feature_sizes)
] ]
processed_data.update(passthrough_data)
return processed_outputs return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+18 -10
View File
@@ -142,16 +142,22 @@ class Phi4SiglipDummyInputsBuilder(
class Phi4SiglipMultiModalProcessor( class Phi4SiglipMultiModalProcessor(
BaseMultiModalProcessor[Phi4SiglipProcessingInfo], BaseMultiModalProcessor[Phi4SiglipProcessingInfo],
): ):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
processed = super()._call_hf_processor( valid_mm_items = mm_items.select(
prompt=prompt, {k for k, c in mm_items.get_all_counts().items() if c > 0}
mm_data=mm_data, )
mm_kwargs=mm_kwargs, mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
processed = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
) )
# The HF processor's tokenizer_image_token() replaces the "<image>" # The HF processor's tokenizer_image_token() replaces the "<image>"
@@ -163,10 +169,12 @@ class Phi4SiglipMultiModalProcessor(
# NOTE: tokenizer.__call__() (not .encode()) must be used so that # NOTE: tokenizer.__call__() (not .encode()) must be used so that
# added/special tokens like <|user|>, <|end|> are kept as single IDs. # added/special tokens like <|user|>, <|end|> are kept as single IDs.
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
new_ids = tokenizer(prompt).input_ids new_ids = tokenizer(prompt_text).input_ids
processed["input_ids"] = torch.tensor([new_ids]) processed["input_ids"] = torch.tensor([new_ids])
return processed processed_data = processed
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+22 -20
View File
@@ -51,12 +51,10 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import ( from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor, BaseMultiModalProcessor,
BaseProcessingInfo, BaseProcessingInfo,
MultiModalProcessingInfo,
ProcessorInputs, ProcessorInputs,
PromptReplacement, PromptReplacement,
PromptUpdate, PromptUpdate,
PromptUpdateDetails, PromptUpdateDetails,
TimingContext,
) )
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors from vllm.sequence import IntermediateTensors
@@ -213,6 +211,9 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]): class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]):
# The tokens are already inserted by the chat template
hf_processor_applies_updates = True
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
hf_inputs: Mapping[str, NestedTensors], hf_inputs: Mapping[str, NestedTensors],
@@ -220,24 +221,35 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
) -> Mapping[str, MultiModalFieldConfig]: ) -> Mapping[str, MultiModalFieldConfig]:
return dict(images=MultiModalFieldConfig.batched("image")) return dict(images=MultiModalFieldConfig.batched("image"))
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
outputs = self.info.ctx.call_hf_processor( outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs), self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt, **mm_data), dict(text=prompt_text, **mm_data),
# Avoid padding issue # Avoid padding issue
dict(**mm_kwargs, return_tensors=None), dict(**hf_processor_mm_kwargs, return_tensors=None),
) )
# Missing batch dimension # Missing batch dimension
if is_list_of(outputs["input_ids"], int): if is_list_of(outputs["input_ids"], int):
outputs["input_ids"] = [outputs["input_ids"]] outputs["input_ids"] = [outputs["input_ids"]]
return outputs processed_data = outputs
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates( def _get_prompt_updates(
self, self,
@@ -273,16 +285,6 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
), ),
] ]
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx)
# NOTE: The tokens are already inserted by the chat template
return prompt_ids, mm_info, True
@MULTIMODAL_REGISTRY.register_processor( @MULTIMODAL_REGISTRY.register_processor(
PixtralMultiModalProcessor, PixtralMultiModalProcessor,
@@ -499,83 +499,6 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
class Qwen2_5OmniThinkerMultiModalProcessor( class Qwen2_5OmniThinkerMultiModalProcessor(
BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo] BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
): ):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
if audios:
# NOTE: Qwen2.5-Omni processor accept "audio"
mm_data["audio"] = audios
mm_kwargs = dict(
**mm_kwargs,
)
elif (
mm_kwargs.get("use_audio_in_video")
and "audio" not in mm_data
and mm_data.get("videos")
):
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
# and populated mm_data["audio"] itself before delegating here,
# in which case this isn't a genuine "no audio" case. Likewise,
# mm_data can be empty on a multimodal-processor-cache hit, where
# there's nothing to (re-)process this call and the real result
# comes from the cache — not a genuine "no audio" case either.
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
if mm_data.get("videos") and (
merged.keys() & {"size", "min_pixels", "max_pixels"}
):
mm_kwargs = dict(mm_kwargs)
video_size = dict(self.info.get_hf_processor().video_processor.size)
size_override = merged.get("size")
if size_override is not None:
video_size = video_size | size_override
min_pixels = merged.get("min_pixels")
if min_pixels is not None:
video_size["shortest_edge"] = min_pixels
max_pixels = merged.get("max_pixels")
if max_pixels is not None:
video_size["longest_edge"] = max_pixels
mm_kwargs["size"] = video_size
hf_inputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
input_features = hf_inputs.pop("input_features", None)
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
if "input_audio_features" not in hf_inputs and input_features is not None:
if feature_attention_mask is not None:
input_features = input_features.permute(0, 2, 1)[
feature_attention_mask.bool()
].permute(1, 0)
hf_inputs["input_audio_features"] = input_features
if (
"audio_feature_lengths" not in hf_inputs
and feature_attention_mask is not None
):
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
if video_second_per_grid is not None:
hf_inputs["second_per_grid_ts"] = video_second_per_grid
use_audio_in_video = mm_kwargs.get("use_audio_in_video", False)
hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
return hf_inputs
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
hf_inputs: BatchFeature, hf_inputs: BatchFeature,
@@ -885,7 +808,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
), ),
] ]
def _apply_hf_processor_mm_only( def _apply_hf_processor_main(
self, self,
mm_items: MultiModalDataItems, mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
@@ -903,13 +826,87 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
) )
mm_counts["audio"] -= mm_counts["video"] mm_counts["audio"] -= mm_counts["video"]
_, mm_processed_data = self._apply_hf_processor_text_mm( valid_mm_items = mm_items.select(
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts), {k for k, c in mm_items.get_all_counts().items() if c > 0}
mm_items=mm_items, )
hf_processor_mm_kwargs=hf_processor_mm_kwargs, processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
mm_data = dict(processor_data)
audios = mm_data.pop("audios", [])
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
if audios:
# NOTE: Qwen2.5-Omni processor accept "audio"
mm_data["audio"] = audios
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
)
elif (
hf_processor_mm_kwargs.get("use_audio_in_video")
and "audio" not in mm_data
and mm_data.get("videos")
):
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
# and populated mm_data["audio"] itself before delegating here,
# in which case this isn't a genuine "no audio" case. Likewise,
# mm_data can be empty on a multimodal-processor-cache hit, where
# there's nothing to (re-)process this call and the real result
# comes from the cache — not a genuine "no audio" case either.
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
if mm_data.get("videos") and (
merged.keys() & {"size", "min_pixels", "max_pixels"}
):
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
video_size = dict(self.info.get_hf_processor().video_processor.size)
size_override = merged.get("size")
if size_override is not None:
video_size = video_size | size_override
min_pixels = merged.get("min_pixels")
if min_pixels is not None:
video_size["shortest_edge"] = min_pixels
max_pixels = merged.get("max_pixels")
if max_pixels is not None:
video_size["longest_edge"] = max_pixels
hf_processor_mm_kwargs["size"] = video_size
hf_inputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
) )
return mm_processed_data input_features = hf_inputs.pop("input_features", None)
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
if "input_audio_features" not in hf_inputs and input_features is not None:
if feature_attention_mask is not None:
input_features = input_features.permute(0, 2, 1)[
feature_attention_mask.bool()
].permute(1, 0)
hf_inputs["input_audio_features"] = input_features
if (
"audio_feature_lengths" not in hf_inputs
and feature_attention_mask is not None
):
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
if video_second_per_grid is not None:
hf_inputs["second_per_grid_ts"] = video_second_per_grid
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
processed_data = hf_inputs
processed_data.update(passthrough_data)
processed_data.pop("input_ids")
return processed_data
class Qwen2_5OmniConditionalGenerationMixin: class Qwen2_5OmniConditionalGenerationMixin:
-10
View File
@@ -1154,16 +1154,6 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True), second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
) )
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_qwen2_5_vl.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates( def _get_prompt_updates(
self, self,
mm_items: MultiModalDataItems, mm_items: MultiModalDataItems,
+12 -21
View File
@@ -235,36 +235,27 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]): class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any], hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> tuple[Mapping[str, object], Mapping[str, object]]:
# NOTE - we rename audios -> audio in mm data because transformers has feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
# deprecated audios for the qwen2audio processor and will remove
# support for it in transformers 4.54. mm_data = dict(mm_data)
audios = mm_data.pop("audios", []) audios = mm_data.pop("audios", [])
if audios: if audios:
mm_data["audio"] = audios mm_data["audio"] = audios
# Text-only input not supported in composite processor hf_processor_mm_kwargs = dict(
if not mm_data.get("audio", []): **hf_processor_mm_kwargs,
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_kwargs = dict(
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate, sampling_rate=feature_extractor.sampling_rate,
) )
return super()._call_hf_processor( return mm_data, hf_processor_mm_kwargs
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
@@ -1195,79 +1195,95 @@ Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder
class Qwen3OmniMoeThinkerMultiModalProcessor( class Qwen3OmniMoeThinkerMultiModalProcessor(
Qwen2_5OmniThinkerMultiModalProcessor, Qwen2_5OmniThinkerMultiModalProcessor,
): ):
def _call_hf_processor( def _get_hf_mm_data(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], ) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_kwargs: Mapping[str, object], processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
def pad_to_hop_length(x: np.ndarray, hop_length: int) -> np.ndarray: audios = processor_data.get("audios")
length = x.shape[-1]
if length % hop_length != 0:
pad_length = hop_length - (length % hop_length)
x = np.pad(x, (0, pad_length), mode="constant", constant_values=0)
return x
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
hop_length = feature_extractor.hop_length
if audios: if audios:
# NOTE: Qwen3-Omni processor accept "audio" feature_extractor = self.info.get_feature_extractor()
hop_length = feature_extractor.hop_length
def pad_to_hop_length(x: np.ndarray) -> np.ndarray:
length = x.shape[-1]
if length % hop_length != 0:
pad_length = hop_length - (length % hop_length)
x = np.pad(x, (0, pad_length), mode="constant", constant_values=0)
return x
# To make sure the cache works with padding=True, we pre-padded # To make sure the cache works with padding=True, we pre-padded
# the audio to multiple of hop_length. # the audio to a multiple of hop_length.
mm_data["audio"] = [ processor_data = dict(processor_data)
pad_to_hop_length(audio, hop_length) processor_data["audios"] = [
pad_to_hop_length(audio)
if isinstance(audio, np.ndarray) if isinstance(audio, np.ndarray)
else (pad_to_hop_length(audio[0], hop_length), audio[1]) else (pad_to_hop_length(audio[0]), audio[1])
for audio in audios for audio in audios
] ]
return processor_data, passthrough_data
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_items.get_all_counts().get("audio", 0):
# TODO(Isotr0py): Remove this patch after upstream fix PR # TODO(Isotr0py): Remove this patch after upstream fix PR
# released and Transformers version update: # released and Transformers version update:
# https://github.com/huggingface/transformers/pull/41473 # https://github.com/huggingface/transformers/pull/41473
mm_kwargs = dict(mm_kwargs) hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {}) hf_processor_mm_kwargs["audio_kwargs"] = dict(
mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {}) hf_processor_mm_kwargs.get("audio_kwargs") or {}
elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"): )
# mm_data can be empty on a multimodal-processor-cache hit, where hf_processor_mm_kwargs["text_kwargs"] = dict(
# there's nothing to (re-)process this call and the real result hf_processor_mm_kwargs.get("text_kwargs") or {}
# comes from the cache — not a genuine "no audio" case.
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
) )
hf_inputs = super()._call_hf_processor( processed_data = super()._apply_hf_processor_main(
prompt=prompt, mm_items,
mm_data=mm_data, hf_processor_mm_kwargs,
mm_kwargs=mm_kwargs,
) )
if ( if (
"audio_feature_lengths" in hf_inputs "audio_feature_lengths" in processed_data
and "feature_attention_mask" in hf_inputs and "feature_attention_mask" in processed_data
and (audios := mm_data.get("audio", []))
): ):
audio_num_frames = [] valid_mm_items = mm_items.select(
for _, audio in enumerate(audios): {k for k, c in mm_items.get_all_counts().items() if c > 0}
audio_length = len(audio[0]) if isinstance(audio, tuple) else len(audio) )
num_frame = ( processor_data, _ = self._get_hf_mm_data(valid_mm_items)
(audio_length // hop_length) audios = processor_data.get("audios", [])
if audio_length % hop_length == 0 if audios:
else (audio_length // hop_length - 1) feature_extractor = self.info.get_feature_extractor(
**hf_processor_mm_kwargs
) )
if mm_kwargs.get("truncation", False): hop_length = feature_extractor.hop_length
num_frame = min(
num_frame, feature_extractor.n_samples // hop_length audio_num_frames = []
for audio in audios:
audio_length = (
len(audio[0]) if isinstance(audio, tuple) else len(audio)
) )
audio_num_frames.append(num_frame) num_frame = (
hf_inputs["feature_attention_mask"] = [ (audio_length // hop_length)
torch.ones(num_frame) for num_frame in audio_num_frames if audio_length % hop_length == 0
] else (audio_length // hop_length - 1)
hf_inputs["audio_feature_lengths"] = torch.tensor(audio_num_frames) )
return hf_inputs if hf_processor_mm_kwargs.get("truncation", False):
num_frame = min(
num_frame, feature_extractor.n_samples // hop_length
)
audio_num_frames.append(num_frame)
processed_data["feature_attention_mask"] = [
torch.ones(num_frame) for num_frame in audio_num_frames
]
processed_data["audio_feature_lengths"] = torch.tensor(audio_num_frames)
return processed_data
def _maybe_apply_prompt_updates( def _maybe_apply_prompt_updates(
self, self,
+35 -22
View File
@@ -1280,12 +1280,21 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
proc_impl = getattr(type(hf_processor), "replace_video_token", None) proc_impl = getattr(type(hf_processor), "replace_video_token", None)
return proc_impl is not None and proc_impl is not mixin_impl return proc_impl is not None and proc_impl is not mixin_impl
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data) mm_data = dict(mm_data)
# Separate video processing from image processing. Because the videos # Separate video processing from image processing. Because the videos
@@ -1314,8 +1323,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# NOTE: a copy of is created to update do_sample_frames, # NOTE: a copy of is created to update do_sample_frames,
# otherwise mm_hash for the object will be incorrect. # otherwise mm_hash for the object will be incorrect.
video_mm_kwargs = dict(**mm_kwargs) video_mm_kwargs = dict(**hf_processor_mm_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs) merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
if merged.keys() & {"size", "min_pixels", "max_pixels"}: if merged.keys() & {"size", "min_pixels", "max_pixels"}:
video_size = dict(self.info.get_video_processor().size) video_size = dict(self.info.get_video_processor().size)
size_override = merged.get("size") size_override = merged.get("size")
@@ -1365,10 +1374,13 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs: if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
video_mm_kwargs["fps"] = None video_mm_kwargs["fps"] = None
video_outputs = super()._call_hf_processor( video_outputs = self.info.ctx.call_hf_processor(
prompt="<|vision_start|><|video_pad|><|vision_end|>", self.info.get_hf_processor(**video_mm_kwargs),
mm_data=video_mm_data, dict(
mm_kwargs=video_mm_kwargs, text="<|vision_start|><|video_pad|><|vision_end|>",
**video_mm_data,
),
video_mm_kwargs,
) )
# Discard HF output input_ids — we use get_video_repl below # Discard HF output input_ids — we use get_video_repl below
@@ -1424,12 +1436,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# fps/num_frames are video-only kwargs already consumed by the loop; # fps/num_frames are video-only kwargs already consumed by the loop;
# exclude them so the text/image processor call below never gets a list. # exclude them so the text/image processor call below never gets a list.
non_video_mm_kwargs = { non_video_mm_kwargs = {
k: v for k, v in mm_kwargs.items() if k not in ("fps", "num_frames") k: v
for k, v in hf_processor_mm_kwargs.items()
if k not in ("fps", "num_frames")
} }
processed_outputs = super()._call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
prompt=prompt, self.info.get_hf_processor(**non_video_mm_kwargs),
mm_data=mm_data, dict(text=prompt_text, **mm_data),
mm_kwargs=non_video_mm_kwargs, non_video_mm_kwargs,
) )
# Replace each placeholder with pre-computed video tokens. # Replace each placeholder with pre-computed video tokens.
@@ -1443,20 +1457,19 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
hf_config.video_token_id, hf_config.video_token_id,
hf_config.vision_end_token_id, hf_config.vision_end_token_id,
] ]
input_ids = processed_outputs.pop("input_ids") input_ids = processed_data.pop("input_ids")
if not isinstance(input_ids, list): if not isinstance(input_ids, list):
input_ids = input_ids.tolist() input_ids = input_ids.tolist()
(prompt_ids,) = input_ids (prompt_ids,) = input_ids
expanded_ids = _replace_video_token_placeholders( expanded_ids = _replace_video_token_placeholders(
prompt_ids, video_target, video_input_ids_lst prompt_ids, video_target, video_input_ids_lst
) )
processed_outputs["input_ids"] = [expanded_ids] processed_data["input_ids"] = [expanded_ids]
combined_outputs = dict( processed_data.update(video_outputs)
processed_outputs, processed_data.update(passthrough_data)
**video_outputs,
) return processed_data
return BatchFeature(combined_outputs)
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
@@ -345,28 +345,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
return mm_fields return mm_fields
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Run the HF processor and unpad inputs."""
try:
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
except ValueError:
if any(mm_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
prompt_ids = self.info.get_tokenizer().encode(prompt)
hf_inputs = transformers.BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, mm_kwargs)
return hf_inputs
def _unpad_audios( def _unpad_audios(
self, self,
hf_inputs: "BatchFeature", hf_inputs: "BatchFeature",
@@ -450,19 +428,6 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
deriving them from updates.""" deriving them from updates."""
return [] return []
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask which modality owns each token of the expanded prompt, which is the
only marking of the tokens an expansion adds around an item."""
if any(mm_data.values()):
mm_data = {**mm_data, "return_mm_token_type_ids": True}
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_token_ids(self, modality: str) -> list[int]: def _get_mm_token_ids(self, modality: str) -> list[int]:
"""Token ids marking where `modality` sits in the prompt, which for some """Token ids marking where `modality` sits in the prompt, which for some
processors differ from the placeholder written into it. processors differ from the placeholder written into it.
@@ -612,24 +577,55 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
with timing_ctx.record("apply_hf_processor"): with timing_ctx.record("apply_hf_processor"):
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
prompt = hf_processor.decode(prompt) prompt_text = hf_processor.decode(prompt)
# Bypass cached processor and always apply to the full set of mm inputs # Bypass cached processor and always apply to the full set of mm inputs
# NOTE: we can't just set caching=False because base class method # NOTE: we can't just set caching=False because base class method
# transforms outputs to `MultiModalKwargs` which is not going to # transforms outputs to `MultiModalKwargs` which is not going to
# work for Transformers. The vision path has logic tied to # work for Transformers. The vision path has logic tied to
# `mm_tokens_per_modality` in _apply_vision() # `mm_tokens_per_modality` in _apply_vision()
prompt_ids, processed_data = self._apply_hf_processor_text_mm( hf_processor_mm_kwargs = {
prompt_text=prompt, **hf_processor_mm_kwargs,
mm_items=mm_items, # HF processors only accept text, and the decoded string already
hf_processor_mm_kwargs={ # contains any special tokens, so don't let them be added again
**hf_processor_mm_kwargs, # (the HF processor call disables `add_special_tokens`).
# HF processors only accept text, and the decoded string already "add_special_tokens": False,
# contains any special tokens, so don't let them be added again }
# (`_call_hf_processor` disables `add_special_tokens`).
"add_special_tokens": False, valid_mm_items = mm_items.select(
}, {k for k, c in mm_items.get_all_counts().items() if c > 0}
) )
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
# Ask which modality owns each token of the expanded prompt, which
# is the only marking of the tokens an expansion adds around an item
if any(processor_data.values()):
processor_data = {**processor_data, "return_mm_token_type_ids": True}
try:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **processor_data),
hf_processor_mm_kwargs,
)
except ValueError:
if any(processor_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
prompt_ids = self.info.get_tokenizer().encode(prompt_text)
processed_data = transformers.BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
self._unpad_images(processed_data)
self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs)
processed_data.update(passthrough_data)
input_ids = processed_data.pop("input_ids")
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
# Use overrides if provided; fallback to data-dependent hashing. # Use overrides if provided; fallback to data-dependent hashing.
with timing_ctx.record("get_mm_hashes"): with timing_ctx.record("get_mm_hashes"):
@@ -773,17 +769,41 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
except (AttributeError, KeyError, TypeError): except (AttributeError, KeyError, TypeError):
return None return None
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> "BatchFeature": ) -> "BatchFeature":
"""Ask for the replacement each placeholder expands to, and record it as mm_counts = mm_items.get_all_counts()
per-item fields: its token ids, and the tokens or patches behind them."""
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
# Ask for the replacement each placeholder expands to, and record it as
# per-item fields: its token ids, and the tokens or patches behind them
if has_mm_data := any(mm_data.values()): if has_mm_data := any(mm_data.values()):
mm_data = {**mm_data, "return_text_replacement_offsets": True} mm_data = {**mm_data, "return_text_replacement_offsets": True}
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
try:
hf_inputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
except ValueError:
if any(mm_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
tokenizer = self.info.get_tokenizer()
hf_inputs = transformers.BatchFeature(
dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)
# Drop the inputs the model would reject # Drop the inputs the model would reject
hf_inputs.pop("mm_token_type_ids", None) hf_inputs.pop("mm_token_type_ids", None)
hf_inputs.pop("token_type_ids", None) hf_inputs.pop("token_type_ids", None)
@@ -802,6 +822,8 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"so it can be fixed, and install `transformers<5.15.0` in the " "so it can be fixed, and install `transformers<5.15.0` in the "
"meantime to locate placeholders in the expanded prompt instead." "meantime to locate placeholders in the expanded prompt instead."
) )
hf_inputs.update(passthrough_data)
hf_inputs.pop("input_ids", None)
return hf_inputs return hf_inputs
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
@@ -829,6 +851,9 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum())) counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
hf_inputs["num_audio_tokens"] = torch.tensor(counts) hf_inputs["num_audio_tokens"] = torch.tensor(counts)
hf_inputs.update(passthrough_data)
hf_inputs.pop("input_ids")
return hf_inputs return hf_inputs
+68 -25
View File
@@ -190,41 +190,84 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]): class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, return self.dummy_inputs.get_dummy_text(mm_counts)
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Text-only input not supported in composite processor
if not mm_data.get("audios", []):
prompt_ids = self.info.get_tokenizer().encode(
prompt, add_special_tokens=False
)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
mm_data = dict(mm_data) def _preprocess_hf_mm_data(
audios = mm_data.pop("audios", []) self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
audios = mm_data.get("audios", [])
assert isinstance(audios, list) assert isinstance(audios, list)
feature_extractor = self.info.get_feature_extractor(**mm_kwargs) feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_kwargs = dict( hf_processor_mm_kwargs = dict(
**mm_kwargs, **hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate, sampling_rate=feature_extractor.sampling_rate,
include_audio_num_chunks=True, include_audio_num_chunks=True,
) )
item_processor_data = dict(**mm_data, audios=audios) return mm_data, hf_processor_mm_kwargs
output = super()._call_hf_processor( def _apply_hf_processor_main(
prompt=prompt, self,
mm_data=item_processor_data, mm_items: MultiModalDataItems,
mm_kwargs=mm_kwargs, hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
) )
output["audio_features"] = output.pop("audio_values") processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
return output if processor_data:
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
processor_data,
hf_processor_mm_kwargs,
)
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
if prompt_text is not None:
processor_data = dict(text=prompt_text, **processor_data)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
def patched_call(**kwargs) -> BatchFeature:
# The remote UltravoxProcessor does not support the
# `truncation` kwarg injected by `call_hf_processor`: it
# passes `truncation` to its audio processor by itself while
# also forwarding `**kwargs` there, causing a duplicate
# keyword argument error. Its hardcoded `truncation=False`
# already matches vLLM's intended behavior.
kwargs.pop("truncation", None)
return hf_processor(**kwargs)
processed_data = self.info.ctx.call_hf_processor(
patched_call,
processor_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
else:
processed_data = BatchFeature(dict(passthrough_data))
return self._postprocess_hf_mm_data(
processor_data,
hf_processor_mm_kwargs,
processed_data,
)
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "audio_values" in processed_data:
processed_data["audio_features"] = processed_data.pop("audio_values")
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+1 -1
View File
@@ -225,7 +225,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):
self, self,
inputs: ProcessorInputs, inputs: ProcessorInputs,
timing_ctx: TimingContext, timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]: ) -> MultiModalProcessingInfo:
# The processor logic differs for single-image (crop) vs multi-image # The processor logic differs for single-image (crop) vs multi-image
# (no crop) requests. The processing cache assumes per-item output is # (no crop) requests. The processing cache assumes per-item output is
# invariant of how many images are passed per prompt, so we only cache # invariant of how many images are passed per prompt, so we only cache
+23 -25
View File
@@ -47,12 +47,10 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import ( from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor, BaseMultiModalProcessor,
BaseProcessingInfo, BaseProcessingInfo,
MultiModalProcessingInfo,
PlaceholderFeaturesInfo, PlaceholderFeaturesInfo,
ProcessorInputs, ProcessorInputs,
PromptReplacement, PromptReplacement,
PromptUpdate, PromptUpdate,
TimingContext,
) )
from vllm.sequence import IntermediateTensors from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config from vllm.tokenizers import cached_tokenizer_from_config
@@ -204,6 +202,9 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]): class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
# The tokens are already inserted by the chat template
hf_processor_applies_updates = True
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
hf_inputs: Mapping[str, NestedTensors], hf_inputs: Mapping[str, NestedTensors],
@@ -220,12 +221,14 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
# skip validation here # skip validation here
pass pass
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data) mm_data = dict(mm_data)
audios = mm_data.pop("audios", []) audios = mm_data.pop("audios", [])
@@ -233,18 +236,23 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
# MistralCommonVoxtralProcessor accepts "audio" # MistralCommonVoxtralProcessor accepts "audio"
mm_data["audio"] = audios mm_data["audio"] = audios
outputs = self.info.ctx.call_hf_processor( # Avoid padding issue
self.info.get_hf_processor(**mm_kwargs), hf_processor_mm_kwargs = dict(**hf_processor_mm_kwargs, return_tensors=None)
dict(text=prompt, **mm_data),
# Avoid padding issue
dict(**mm_kwargs, return_tensors=None),
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
# Missing batch dimension # Missing batch dimension
if is_list_of(outputs["input_ids"], int): input_ids = processed_data.get("input_ids")
outputs["input_ids"] = [outputs["input_ids"]] if is_list_of(input_ids, int):
processed_data["input_ids"] = [input_ids]
return outputs return processed_data
def _get_prompt_updates( def _get_prompt_updates(
self, self,
@@ -287,16 +295,6 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
), ),
] ]
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx)
# NOTE: The tokens are already inserted by the chat template
return prompt_ids, mm_info, True
@MULTIMODAL_REGISTRY.register_processor( @MULTIMODAL_REGISTRY.register_processor(
VoxtralMultiModalProcessor, VoxtralMultiModalProcessor,
+27 -18
View File
@@ -756,27 +756,36 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
# for encoder profiling. # for encoder profiling.
return [0] return [0]
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self, self,
prompt: str,
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> tuple[Mapping[str, object], Mapping[str, object]]:
if mm_data: feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_data = dict(audio=mm_data.pop("audios")) mm_data = dict(mm_data)
mm_kwargs = dict( mm_data["audio"] = mm_data.pop("audios")
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate, hf_processor_mm_kwargs = dict(
) **hf_processor_mm_kwargs,
processed_outputs = super()._call_hf_processor( sampling_rate=feature_extractor.sampling_rate,
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
) )
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels") return mm_data, hf_processor_mm_kwargs
return processed_outputs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+2 -7
View File
@@ -649,13 +649,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
processor_data["videos"] = raw_videos processor_data["videos"] = raw_videos
return processor_data, passthrough_data return processor_data, passthrough_data
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
self, return self.dummy_inputs.get_dummy_text(mm_counts)
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+18 -14
View File
@@ -22,7 +22,7 @@ from vllm.multimodal.inputs import (
MultiModalFieldConfig, MultiModalFieldConfig,
MultiModalKwargsItems, MultiModalKwargsItems,
) )
from vllm.multimodal.parse import MultiModalDataParser from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.processing import ( from vllm.multimodal.processing import (
BaseDummyInputsBuilder, BaseDummyInputsBuilder,
BaseMultiModalProcessor, BaseMultiModalProcessor,
@@ -135,8 +135,8 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
num_images = mm_counts.get("image", 0) num_images = mm_counts.get("image", 0)
num_audios = mm_counts.get("audio", 0) num_audios = mm_counts.get("audio", 0)
# Use spellings the renderer would emit; tokenization is bypassed in # Use spellings the renderer would emit; tokenization is bypassed in
# _call_hf_processor (we build input_ids directly), so the exact text # _apply_hf_processor_main (we build input_ids directly), so the exact
# only needs to be a stable per-item marker. # text only needs to be a stable per-item marker.
return ("<|content_image|>" * num_images) + ( return ("<|content_image|>" * num_images) + (
"<|content_audio_input|>" * num_audios "<|content_audio_input|>" * num_audios
) )
@@ -179,17 +179,19 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]): class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
# Inkling is not a standard HF processor (no fused text+mm call), so we run valid_mm_items = mm_items.select(
# the vendored extractors ourselves and tokenize the text separately. {k for k, c in mm_items.get_all_counts().items() if c > 0}
# The MM placeholders in `prompt` are expanded later by the prompt )
# updates, so here we emit ONE placeholder id per media item. mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
processor = self.info.get_hf_processor(**mm_kwargs)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
images = mm_data.get("images") or [] images = mm_data.get("images") or []
@@ -200,7 +202,7 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
audios = list(cast(Iterable[Any], audios)) audios = list(cast(Iterable[Any], audios))
prompt_ids = self._tokenize_with_placeholders( prompt_ids = self._tokenize_with_placeholders(
prompt, tokenizer, len(images), len(audios) prompt_text, tokenizer, len(images), len(audios)
) )
data: dict[str, Any] = {"input_ids": [prompt_ids]} data: dict[str, Any] = {"input_ids": [prompt_ids]}
@@ -232,7 +234,9 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
data["input_audio_features"] = input_audio_features data["input_audio_features"] = input_audio_features
data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64) data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)
return BatchFeature(data=data, tensor_type=None) processed_data = BatchFeature(data=data, tensor_type=None)
processed_data.update(passthrough_data)
return processed_data
def _tokenize_with_placeholders( def _tokenize_with_placeholders(
self, self,
@@ -231,19 +231,6 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]): class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
"""Image-only multi-modal processor for Kimi-K3.""" """Image-only multi-modal processor for Kimi-K3."""
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override so the base always routes through the text+mm path
# (`KimiK3Processor.__call__`). Otherwise the mm-only fast path calls
# the checkpoint image processor directly with bare PIL images, but it
# requires `{"type": "image", "image": PIL}` media dicts that only our
# wrapper builds.
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
hf_inputs: BatchFeature, hf_inputs: BatchFeature,
+19 -8
View File
@@ -306,12 +306,21 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing
class MiniMaxM3VLMultiModalProcessor( class MiniMaxM3VLMultiModalProcessor(
BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo] BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo]
): ):
def _call_hf_processor( def _apply_hf_processor_main(
self, self,
prompt: str, mm_items: MultiModalDataItems,
mm_data: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature: ) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data) mm_data = dict(mm_data)
# With ``video_needs_metadata=True`` each video arrives as a # With ``video_needs_metadata=True`` each video arrives as a
# ``(frames, metadata)`` tuple. Split the frames back out and forward the # ``(frames, metadata)`` tuple. Split the frames back out and forward the
@@ -340,15 +349,17 @@ class MiniMaxM3VLMultiModalProcessor(
# Override the video processor's default do_resize=False (set for a # Override the video processor's default do_resize=False (set for a
# pre-resized pipeline) to True for vLLM's raw-frame inputs. # pre-resized pipeline) to True for vLLM's raw-frame inputs.
merged = dict(do_resize=True, **mm_kwargs) merged = dict(do_resize=True, **hf_processor_mm_kwargs)
data = dict(text=prompt, **mm_data) data = dict(text=prompt_text, **mm_data)
if video_metadata is not None: if video_metadata is not None:
data["video_metadata"] = video_metadata data["video_metadata"] = video_metadata
return self.info.ctx.call_hf_processor( processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs), self.info.get_hf_processor(**hf_processor_mm_kwargs),
data, data,
merged, merged,
) )
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+6 -3
View File
@@ -262,9 +262,6 @@ class InputProcessingContext:
hf_processor: Callable[..., BatchFeature] | ProcessorMixin, hf_processor: Callable[..., BatchFeature] | ProcessorMixin,
data: Mapping[str, object], data: Mapping[str, object],
kwargs: Mapping[str, object] = {}, kwargs: Mapping[str, object] = {},
*,
num_tries: int = 1,
max_tries: int = 5,
) -> BatchFeature: ) -> BatchFeature:
""" """
Call `hf_processor` on the prompt `data` Call `hf_processor` on the prompt `data`
@@ -274,6 +271,12 @@ class InputProcessingContext:
merged_kwargs = self.get_merged_mm_kwargs(kwargs) merged_kwargs = self.get_merged_mm_kwargs(kwargs)
# vLLM needs the full untruncated sequence to keep multi-modal
# placeholder tokens aligned; note that the text inputs in
# call_hf_processor are just dummy text, not the original prompt.
# The original prompt is already tokenized by the renderer.
merged_kwargs.setdefault("truncation", False)
allowed_kwargs = get_allowed_kwarg_only_overrides( allowed_kwargs = get_allowed_kwarg_only_overrides(
hf_processor, hf_processor,
merged_kwargs, merged_kwargs,
+1 -4
View File
@@ -91,10 +91,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
# prompt tokens. # prompt tokens.
dummy_prompt = [] dummy_prompt = []
else: else:
dummy_prompt = tokenizer.encode( dummy_prompt = tokenizer.encode(dummy_text, truncation=False)
dummy_text,
**self.info.default_tok_params.get_encode_kwargs(),
)
return ProcessorInputs( return ProcessorInputs(
prompt=dummy_prompt, prompt=dummy_prompt,
+99 -137
View File
@@ -5,9 +5,10 @@ from collections import defaultdict, deque
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
from dataclasses import dataclass, field, replace from dataclasses import dataclass, field, replace
from enum import Enum from enum import Enum
from functools import lru_cache, partial from functools import lru_cache
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
ClassVar,
Generic, Generic,
NamedTuple, NamedTuple,
Protocol, Protocol,
@@ -28,7 +29,6 @@ from vllm.inputs import (
) )
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike from vllm.tokenizers import TokenizerLike
from vllm.transformers_utils.processor import call_hf_processor_mm_only
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
from ..inputs import ( from ..inputs import (
@@ -1177,6 +1177,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
Not to be confused with `transformers.ProcessorMixin`. Not to be confused with `transformers.ProcessorMixin`.
""" """
hf_processor_applies_updates: ClassVar[bool] = False
"""
Only set to True if the tokenizer or chat template
(which are run before MM processing) inserts placeholder tokens.
"""
def __init__( def __init__(
self, self,
info: _I, info: _I,
@@ -1302,131 +1308,100 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
return processor_data, passthrough_data return processor_data, passthrough_data
def _call_hf_processor( def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None:
"""
Get the text to pass to the HF processor alongside the multi-modal
data.
By default, no text is passed. If the HF processor requires that
text and multi-modal items correspond to each other, you should
override this method to return dummy text generated by
[`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder].
"""
return None
def _preprocess_hf_mm_data(
self, self,
prompt: str,
# Not to be confused with `mm_data` in `self.apply`.
# This refers to the data to be passed to HF processor.
mm_data: Mapping[str, object], mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""
Pre-process the multi-modal data and HF processor keyword arguments
before they are passed to the HF processor.
By default, both are returned as-is. If the HF processor expects the
multi-modal data under different keys than those provided by the
multi-modal items (e.g. `audio` instead of `audios`), or requires
additional keyword arguments (e.g. `sampling_rate`), you should
override this method.
"""
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature: ) -> BatchFeature:
""" """
Call the HF processor on the prompt text and Post-process the output of the HF processor.
associated multi-modal data.
By default, the output is returned as-is. If you need to modify the
output of the HF processor before it is converted into multi-modal
keyword arguments, you should override this method.
""" """
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
mm_kwargs,
)
def _apply_hf_processor_text_mm(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[list[int], BatchFeature]:
"""
Apply the HF processor on the prompt text and multi-modal data
together.
"""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
processed_data = self._call_hf_processor(
prompt=prompt_text,
mm_data=processor_data,
mm_kwargs=hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
input_ids = processed_data.pop("input_ids")
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
return prompt_ids, processed_data
def _apply_hf_processor_tokens_only(
self,
prompt_tokens: list[int],
) -> list[int]:
"""
Apply the HF processor on the prompt tokens only.
Most HF processors accept prompt text but not prompt tokens.
If the HF processor adds or removes tokens that are not related to
multi-modal data, you should override this method so it is consistent
with the tokenization of the corresponding text.
"""
return prompt_tokens
def _apply_hf_processor_mm_only(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Apply the HF processor on the multi-modal data only.
Since HF processor requires that text and multi-modal items
correspond to each other, we generate dummy text using
[`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
to go along with the multi-modal data.
"""
# Custom logic based on text inputs
if type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor:
mm_counts = mm_items.get_all_counts()
_, mm_processed_data = self._apply_hf_processor_text_mm(
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
return mm_processed_data
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
processed_data = self.info.ctx.call_hf_processor(
partial(
call_hf_processor_mm_only,
self.info.get_hf_processor(**hf_processor_mm_kwargs),
),
processor_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data return processed_data
def _apply_hf_processor_main( def _apply_hf_processor_main(
self, self,
prompt: list[int],
mm_items: MultiModalDataItems, mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[list[int], BatchFeature, bool]: ) -> BatchFeature:
""" """
Apply the HF processor on the prompt tokens and multi-modal data. Apply the HF processor on the multi-modal data.
In addition, return whether prompt updates have been applied
(`False` here, since for token prompts the updates are normally
applied separately via `_maybe_apply_prompt_updates`).
""" """
prompt_ids = self._apply_hf_processor_tokens_only(prompt) valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
mm_processed_data = self._apply_hf_processor_mm_only( if processor_data:
mm_items=mm_items, processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
hf_processor_mm_kwargs=hf_processor_mm_kwargs, processor_data, hf_processor_mm_kwargs
)
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
if prompt_text is not None:
processor_data = dict(text=prompt_text, **processor_data)
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
processor_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
else:
from transformers.feature_extraction_utils import BatchFeature
processed_data = BatchFeature(dict(passthrough_data))
return self._postprocess_hf_mm_data(
processor_data,
hf_processor_mm_kwargs,
processed_data,
) )
return prompt_ids, mm_processed_data, False def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
"""
Post-process the prompt token IDs before locating or applying
multi-modal prompt updates.
By default, the prompt is returned as-is. If the HF processor (or
chat template) applies additional transformations to the prompt
that are not reflected in its multi-modal outputs, you should
override this method to replicate them.
"""
return prompt
def _get_cache_missing_items( def _get_cache_missing_items(
self, self,
@@ -1531,14 +1506,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self, self,
inputs: ProcessorInputs, inputs: ProcessorInputs,
timing_ctx: TimingContext, timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]: ) -> MultiModalProcessingInfo:
with timing_ctx.record("apply_hf_processor"): with timing_ctx.record("apply_hf_processor"):
( mm_processed_data = self._apply_hf_processor_main(
prompt_ids,
mm_processed_data,
is_update_applied,
) = self._apply_hf_processor_main(
prompt=inputs.prompt,
mm_items=inputs.mm_data_items, mm_items=inputs.mm_data_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs, hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
) )
@@ -1569,13 +1539,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt_updates=mm_prompt_updates, prompt_updates=mm_prompt_updates,
) )
return prompt_ids, mm_info, is_update_applied return mm_info
def _cached_apply_hf_processor( def _cached_apply_hf_processor(
self, self,
inputs: ProcessorInputs, inputs: ProcessorInputs,
timing_ctx: TimingContext, timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]: ) -> MultiModalProcessingInfo:
""" """
Apply the HF processor on the full prompt text, Apply the HF processor on the full prompt text,
caching the results and reusing cached results. caching the results and reusing cached results.
@@ -1599,16 +1569,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_hashes=mm_hashes, mm_hashes=mm_hashes,
) )
# NOTE: `prompt` does not correspond to `mm_missing_data_items`, # NOTE: The prompt does not correspond to `mm_missing_data_items`,
# so we can't apply prompt updates until the new multimodal # so we can't apply prompt updates until the new multimodal
# items are combined with the cached multimodal items # items are combined with the cached multimodal items
with timing_ctx.record("apply_hf_processor"): with timing_ctx.record("apply_hf_processor"):
( mm_missing_processed_data = self._apply_hf_processor_main(
prompt_ids,
mm_missing_processed_data,
is_update_applied,
) = self._apply_hf_processor_main(
prompt=inputs.prompt,
mm_items=mm_missing_data_items, mm_items=mm_missing_data_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs, hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
) )
@@ -1641,7 +1606,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt_updates=mm_prompt_updates, prompt_updates=mm_prompt_updates,
) )
return prompt_ids, mm_info, is_update_applied return mm_info
def _apply_token_matches( def _apply_token_matches(
self, self,
@@ -1770,7 +1735,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"There is likely a problem with your " "There is likely a problem with your "
"implementation of merged multi-modal processor for this " "implementation of merged multi-modal processor for this "
"model (usually arising from an inconsistency between " "model (usually arising from an inconsistency between "
"`_call_hf_processor` and `_get_mm_fields_config`)." "`_apply_hf_processor_main` and `_get_mm_fields_config`)."
) )
def _validate_mm_updates( def _validate_mm_updates(
@@ -1805,8 +1770,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
f"Expected there to be {item_count} prompt placeholders " f"Expected there to be {item_count} prompt placeholders "
f"corresponding to {item_count} {modality} items, but " f"corresponding to {item_count} {modality} items, but "
f"instead found {len(placeholders)} prompt placeholders! " f"instead found {len(placeholders)} prompt placeholders! "
"Make sure the implementation of `_call_hf_processor` and " "Make sure the implementation of `_apply_hf_processor_main` "
"`_get_mm_fields_config` are consistent with each other." "and `_get_mm_fields_config` are consistent with each other."
) )
def _maybe_apply_prompt_updates( def _maybe_apply_prompt_updates(
@@ -1854,11 +1819,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
3. Extract information about the placeholder tokens from the 3. Extract information about the placeholder tokens from the
processed token IDs. processed token IDs.
""" """
( prompt_ids = self._postprocess_prompt(inputs.prompt)
prompt_ids, mm_info = self._cached_apply_hf_processor(inputs, timing_ctx)
mm_info,
is_update_applied,
) = self._cached_apply_hf_processor(inputs, timing_ctx)
with timing_ctx.record("apply_prompt_updates"): with timing_ctx.record("apply_prompt_updates"):
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates( prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
@@ -1866,7 +1828,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt_ids=prompt_ids, prompt_ids=prompt_ids,
mm_kwargs=mm_info.kwargs, mm_kwargs=mm_info.kwargs,
mm_prompt_updates=mm_info.prompt_updates, mm_prompt_updates=mm_info.prompt_updates,
is_update_applied=is_update_applied, is_update_applied=self.hf_processor_applies_updates,
) )
mm_placeholder_ranges = { mm_placeholder_ranges = {
-44
View File
@@ -11,17 +11,13 @@ from transformers import (
AutoImageProcessor, AutoImageProcessor,
AutoProcessor, AutoProcessor,
AutoVideoProcessor, AutoVideoProcessor,
BatchFeature,
processing_utils, processing_utils,
) )
from transformers.audio_utils import AudioInput
from transformers.feature_extraction_utils import FeatureExtractionMixin from transformers.feature_extraction_utils import FeatureExtractionMixin
from transformers.image_processing_utils import BaseImageProcessor from transformers.image_processing_utils import BaseImageProcessor
from transformers.image_utils import ImageInput
from transformers.models.auto.video_processing_auto import VIDEO_PROCESSOR_MAPPING_NAMES from transformers.models.auto.video_processing_auto import VIDEO_PROCESSOR_MAPPING_NAMES
from transformers.processing_utils import ProcessorMixin from transformers.processing_utils import ProcessorMixin
from transformers.video_processing_utils import BaseVideoProcessor from transformers.video_processing_utils import BaseVideoProcessor
from transformers.video_utils import VideoInput
from typing_extensions import TypeVar from typing_extensions import TypeVar
from vllm.logger import init_logger from vllm.logger import init_logger
@@ -554,43 +550,3 @@ def cached_video_processor_from_config(
processor_cls_overrides=processor_cls, # type: ignore[arg-type] processor_cls_overrides=processor_cls, # type: ignore[arg-type]
**_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs), **_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs),
) )
def call_hf_processor_mm_only(
processor: ProcessorMixin,
images: ImageInput | None = None,
videos: VideoInput | None = None,
audio: AudioInput | None = None,
**kwargs,
) -> BatchFeature:
output_kwargs = processor._merge_kwargs(
get_processor_kwargs_type(processor),
**kwargs,
)
if audio is not None and (
feature_extractor := getattr(processor, "feature_extractor", None)
):
audio_inputs = feature_extractor(audio, **output_kwargs["audio_kwargs"])
audio_inputs["feature_attention_mask"] = audio_inputs.pop("attention_mask")
else:
audio_inputs = {}
if images is not None and (
image_processor := getattr(processor, "image_processor", None)
):
images_inputs = image_processor(images=images, **output_kwargs["images_kwargs"])
else:
images_inputs = {}
if videos is not None and (
video_processor := getattr(processor, "video_processor", None)
):
videos_inputs = video_processor(videos=videos, **output_kwargs["videos_kwargs"])
else:
videos_inputs = {}
return BatchFeature(
data={**audio_inputs, **images_inputs, **videos_inputs},
tensor_type=kwargs.get("return_tensors"),
)