[MM] Simplify _apply_hf_processor_main (#53275)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -153,11 +153,13 @@ def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
|||||||
|
|
||||||
## 3. Specify dummy inputs
|
## 3. Specify dummy inputs
|
||||||
|
|
||||||
Then, inherit [BaseDummyInputsBuilder][vllm.multimodal.processing.BaseDummyInputsBuilder] to construct dummy inputs for
|
Then, inherit [BaseDummyInputsBuilder][vllm.multimodal.processing.BaseDummyInputsBuilder] to construct the dummy inputs
|
||||||
HF processing. The processed outputs are also used for memory profiling.
|
that are used for memory profiling.
|
||||||
|
|
||||||
Override the abstract methods [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text] and [get_dummy_mm_data][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_mm_data] to construct dummy inputs. These dummy inputs should result in the worst-case memory usage of the model so that vLLM can reserve the correct amount of memory for it.
|
Override the abstract methods [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text] and [get_dummy_mm_data][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_mm_data] to construct dummy inputs. These dummy inputs should result in the worst-case memory usage of the model so that vLLM can reserve the correct amount of memory for it.
|
||||||
|
|
||||||
|
Besides profiling, the dummy text is passed to the HF processor by models whose HF processor requires text corresponding to the multi-modal items (see [Multi-modal fields](#multi-modal-fields)). If the HF processor can process multi-modal data without any text, the dummy text is not used during HF processing.
|
||||||
|
|
||||||
Assuming that the memory usage increases with the number of tokens, the dummy inputs can be constructed to maximize the number of output embeddings, which is the same number as placeholder feature tokens.
|
Assuming that the memory usage increases with the number of tokens, the dummy inputs can be constructed to maximize the number of output embeddings, which is the same number as placeholder feature tokens.
|
||||||
|
|
||||||
=== "Basic example: LLaVA"
|
=== "Basic example: LLaVA"
|
||||||
@@ -446,37 +448,64 @@ return a schema of the tensors outputted by the HF processor that are related to
|
|||||||
like in LLaVA, each image's features must be independent of the others (which
|
like in LLaVA, each image's features must be independent of the others (which
|
||||||
is also required for prefix caching to work correctly). So, we un-pad each image
|
is also required for prefix caching to work correctly). So, we un-pad each image
|
||||||
back to its own size by overriding
|
back to its own size by overriding
|
||||||
[BaseMultiModalProcessor._call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor]:
|
[BaseMultiModalProcessor._postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data]
|
||||||
|
to post-process the outputs of the HF processor:
|
||||||
|
|
||||||
??? code
|
??? code
|
||||||
|
|
||||||
```python
|
```python
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
# Mistral3Processor requires text corresponding to the images
|
||||||
prompt: str,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
pixel_values = processed_outputs.get("pixel_values")
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if not mm_data:
|
||||||
|
return processed_data
|
||||||
|
|
||||||
|
pixel_values = processed_data.get("pixel_values")
|
||||||
if pixel_values is not None:
|
if pixel_values is not None:
|
||||||
# Avoid padding since we need the output for each image to be
|
# Avoid padding since we need the output for each image to be
|
||||||
# independent of other images for the cache to work correctly
|
# independent of other images for the cache to work correctly
|
||||||
image_sizes = processed_outputs["image_sizes"]
|
image_sizes = processed_data["image_sizes"]
|
||||||
assert len(pixel_values) == len(image_sizes)
|
assert len(pixel_values) == len(image_sizes)
|
||||||
|
|
||||||
processed_outputs["pixel_values"] = [
|
processed_data["pixel_values"] = [
|
||||||
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
|
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
|
||||||
]
|
]
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The default implementation of
|
||||||
|
[_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]
|
||||||
|
calls the HF processor on the multi-modal data without passing any text.
|
||||||
|
If the HF processor instead requires text corresponding to the multi-modal items,
|
||||||
|
you should override
|
||||||
|
[_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text]
|
||||||
|
to return the dummy text from
|
||||||
|
[BaseDummyInputsBuilder.get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text]
|
||||||
|
like in the example above. If the HF processor expects the multi-modal data
|
||||||
|
under different keys than those provided by the multi-modal items
|
||||||
|
(e.g. `audio` instead of `audios`), or requires additional keyword arguments
|
||||||
|
(e.g. `sampling_rate`), you should override
|
||||||
|
[_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data].
|
||||||
|
If you need to modify the output of the HF processor,
|
||||||
|
you should override
|
||||||
|
[_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data].
|
||||||
|
If the HF processor applies additional transformations to the prompt tokens
|
||||||
|
that are not reflected in its multi-modal outputs (e.g. prepending a BOS token),
|
||||||
|
you should override
|
||||||
|
[_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt]
|
||||||
|
to replicate them.
|
||||||
|
For even more control over how the HF processor is called, you can override
|
||||||
|
[_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]
|
||||||
|
directly.
|
||||||
|
|
||||||
Since `pixel_values` is now a list with one tensor per image, we can override
|
Since `pixel_values` is now a list with one tensor per image, we can override
|
||||||
[_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows:
|
[_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows:
|
||||||
|
|
||||||
@@ -680,7 +709,7 @@ Examples:
|
|||||||
|
|
||||||
### Handling prompt updates unrelated to multi-modal data
|
### Handling prompt updates unrelated to multi-modal data
|
||||||
|
|
||||||
[_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates] assumes that each application of prompt update corresponds to one multi-modal item. If the HF processor performs additional processing regardless of how many multi-modal items there are, you should override [_apply_hf_processor_tokens_only][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_tokens_only] so that the processed token inputs are consistent with the result of applying the HF processor on text inputs. This is because token inputs bypass the HF processor according to [our design](../../design/mm_processing.md).
|
[_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates] assumes that each application of prompt update corresponds to one multi-modal item. If the HF processor performs additional processing on the prompt regardless of how many multi-modal items there are, you should override [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt] so that the prompt token IDs are consistent with the result of applying the HF processor on text inputs. This is because token inputs bypass the HF processor according to [our design](../../design/mm_processing.md).
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
|
|
||||||
@@ -690,7 +719,7 @@ Examples:
|
|||||||
|
|
||||||
### Custom HF processor
|
### Custom HF processor
|
||||||
|
|
||||||
Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and pass it to [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor].
|
Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. It is then applied to the multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor] inside [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main].
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,15 @@
|
|||||||
|
|
||||||
To enable various optimizations in vLLM such as [chunked prefill](../configuration/optimization.md#chunked-prefill) and [prefix caching](../features/automatic_prefix_caching.md), we use [BaseMultiModalProcessor][vllm.multimodal.processing.BaseMultiModalProcessor] to provide the correspondence between placeholder feature tokens (e.g. `<image>`) and multi-modal inputs (e.g. the raw input image) based on the outputs of HF processor.
|
To enable various optimizations in vLLM such as [chunked prefill](../configuration/optimization.md#chunked-prefill) and [prefix caching](../features/automatic_prefix_caching.md), we use [BaseMultiModalProcessor][vllm.multimodal.processing.BaseMultiModalProcessor] to provide the correspondence between placeholder feature tokens (e.g. `<image>`) and multi-modal inputs (e.g. the raw input image) based on the outputs of HF processor.
|
||||||
|
|
||||||
Here are the main features of [BaseMultiModalProcessor][vllm.multimodal.processing.BaseMultiModalProcessor]:
|
In vLLM's rendering pipeline (see [BaseRenderer][vllm.renderers.base.BaseRenderer]), tokenization is performed as a separate step before multi-modal processing. Therefore, `BaseMultiModalProcessor` needs to recreate the output of calling HF processor end-to-end, while not being able to see the original text. This is achieved through **Dummy Input Text** and **Prompt Update Detection**.
|
||||||
|
|
||||||
|
## Dummy Input Text
|
||||||
|
|
||||||
|
Since Transformers 5.10, `ProcessorMixin` now allows multi-modal inputs to be passed by themselves. However, certain subclasses (such as `ChameleonProcessor`) and older out-of-tree implementations may still define their own `__call__` method that assumes the presence of text with corresponding placeholder tokens. This causes a problem as we don't have the original text anymore to pass to these processors.
|
||||||
|
|
||||||
|
To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text] returns so that [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] passes it to the HF processor together with the multi-modal inputs to obtain the processed multi-modal data.
|
||||||
|
|
||||||
|
Similarly, since the multi-modal data extracted by vLLM may not match what a specific HF processor expects, [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] allows each model to adapt the inputs via [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data] (e.g. renaming keys like `audios` to `audio` or injecting extra keyword arguments such as `sampling_rate`) and the outputs via [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data], without having to reimplement the entire method.
|
||||||
|
|
||||||
## Prompt Update Detection
|
## Prompt Update Detection
|
||||||
|
|
||||||
@@ -13,46 +21,9 @@ One of the main responsibilities of HF processor is to update the prompt with pl
|
|||||||
|
|
||||||
The information about which tokens have been updated is key to finding the correspondence between placeholder feature tokens and multi-modal inputs.
|
The information about which tokens have been updated is key to finding the correspondence between placeholder feature tokens and multi-modal inputs.
|
||||||
|
|
||||||
In vLLM, this information is specified using [PromptUpdate][vllm.multimodal.processing.PromptUpdate] in [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates]. We can automatically detect whether HF has updated the prompt by checking the existence of the updated tokens.
|
Since we call HF processor without the input text, we have to perform this update by ourselves. In vLLM, we represent the necessary information using [PromptUpdate][vllm.multimodal.processing.PromptUpdate] in [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates], and apply them via [_apply_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._apply_prompt_updates].
|
||||||
|
|
||||||
## Tokenized Prompt Inputs
|
Some HF processors additionally transform the prompt itself regardless of the multi-modal inputs (such as `ChameleonProcessor` appending a sep token for chat mode). Since the prompt tokens likewise bypass the HF processor, such transformations are replicated via [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt] before the prompt updates are located or applied.
|
||||||
|
|
||||||
To enable tokenization in a separate process, we support passing input token IDs alongside multi-modal data.
|
|
||||||
|
|
||||||
### The problem
|
|
||||||
|
|
||||||
Consider that HF processors follow these main steps:
|
|
||||||
|
|
||||||
1. Tokenize the text
|
|
||||||
2. Process multi-modal inputs
|
|
||||||
3. Perform prompt updates
|
|
||||||
|
|
||||||
And we require that:
|
|
||||||
|
|
||||||
- For text + multi-modal inputs, apply all steps 1--3.
|
|
||||||
- For tokenized + multi-modal inputs, apply only steps 2--3.
|
|
||||||
|
|
||||||
How can we achieve this without rewriting HF processors? We can try to call the HF processor several times on different inputs:
|
|
||||||
|
|
||||||
- For text + multi-modal inputs, simply call the HF processor directly.
|
|
||||||
- For tokenized + multi-modal inputs, call the processor only on the multi-modal inputs.
|
|
||||||
|
|
||||||
While HF processors support text + multi-modal inputs natively, this is not so for tokenized + multi-modal inputs: an error is thrown if the number of input placeholder tokens do not correspond to the number of multi-modal inputs.
|
|
||||||
|
|
||||||
Moreover, since the tokenized text has not passed through the HF processor, we have to apply Step 3 by ourselves to keep the output tokens and multi-modal data consistent with each other.
|
|
||||||
|
|
||||||
### Dummy text
|
|
||||||
|
|
||||||
We work around the first issue by requiring each model to define how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text]. This lets us generate dummy text corresponding to the multi-modal inputs and input them together to obtain the processed multi-modal data.
|
|
||||||
|
|
||||||
### Automatic prompt updating
|
|
||||||
|
|
||||||
We address the second issue by implementing model-agnostic code in
|
|
||||||
[_apply_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._apply_prompt_updates] to automatically update the prompt with feature placeholder tokens based on the specification outputted by [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates].
|
|
||||||
|
|
||||||
### Summary
|
|
||||||
|
|
||||||
With the help of dummy text and automatic prompt updating, our multi-modal processor can finally accept both text and token prompts with multi-modal data. The detailed logic is shown in [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main].
|
|
||||||
|
|
||||||
## Processor Output Caching
|
## Processor Output Caching
|
||||||
|
|
||||||
@@ -60,8 +31,6 @@ Some HF processors, such as the one for Qwen2-VL, are [very slow](https://github
|
|||||||
|
|
||||||
When new data is passed in, we first check which items are in the cache, and which ones are missing. The missing items are passed into the HF processor in a single batch and cached, before being merged with the existing items in the cache.
|
When new data is passed in, we first check which items are in the cache, and which ones are missing. The missing items are passed into the HF processor in a single batch and cached, before being merged with the existing items in the cache.
|
||||||
|
|
||||||
Since we only process the missing multi-modal data items, the number of input placeholder tokens no longer corresponds to the number of the multi-modal inputs, so they can't be passed alongside the text prompt to HF processor. Therefore, we process the text and multi-modal inputs separately, using [dummy text](#dummy-text) to avoid HF errors. Since this skips HF's prompt updating code, we apply [automatic prompt updating](#automatic-prompt-updating) afterwards to keep the output tokens and multi-modal data consistent with each other.
|
|
||||||
|
|
||||||
## Speeding Up Multi‑Modal Data Processing
|
## Speeding Up Multi‑Modal Data Processing
|
||||||
|
|
||||||
### Fused Normalisation on the Device
|
### Fused Normalisation on the Device
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ requests >= 2.26.0
|
|||||||
tqdm
|
tqdm
|
||||||
blake3
|
blake3
|
||||||
py-cpuinfo
|
py-cpuinfo
|
||||||
transformers >= 5.5.3
|
transformers >= 5.10.4
|
||||||
huggingface_hub >= 1.28.0
|
huggingface_hub >= 1.28.0
|
||||||
tokenizers >= 0.21.1 # Required for fast incremental detokenization.
|
tokenizers >= 0.21.1 # Required for fast incremental detokenization.
|
||||||
safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611
|
safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611
|
||||||
|
|||||||
@@ -136,7 +136,7 @@ def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
|
|||||||
# no audio track" case. (The audio limit above is 1, not 0, so that
|
# no audio track" case. (The audio limit above is 1, not 0, so that
|
||||||
# `mm_counts["audio"]` is still declared for this request; a 0 limit
|
# `mm_counts["audio"]` is still declared for this request; a 0 limit
|
||||||
# would make vLLM omit the "audio" key from `mm_counts` entirely, which
|
# would make vLLM omit the "audio" key from `mm_counts` entirely, which
|
||||||
# trips an unrelated assertion in `_apply_hf_processor_mm_only`.)
|
# trips an unrelated assertion in `_apply_hf_processor_main`.)
|
||||||
mm_data = {"video": [video]}
|
mm_data = {"video": [video]}
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="doesn't have audio track"):
|
with pytest.raises(ValueError, match="doesn't have audio track"):
|
||||||
|
|||||||
@@ -105,9 +105,9 @@ def test_audio_chunk_counting(mock_ctx):
|
|||||||
audio_2 = np.zeros(75 * sr)
|
audio_2 = np.zeros(75 * sr)
|
||||||
|
|
||||||
mm_data = {"audio": [audio_1, audio_2]}
|
mm_data = {"audio": [audio_1, audio_2]}
|
||||||
prompt = "<|user|>Listen.<|end|>"
|
|
||||||
|
|
||||||
processed = processor._call_hf_processor(prompt, mm_data, {})
|
mm_items = processor.info.parse_mm_data(mm_data, validate=False)
|
||||||
|
processed = processor._apply_hf_processor_main(mm_items, {})
|
||||||
|
|
||||||
chunk_counts = processed["chunk_counts"]
|
chunk_counts = processed["chunk_counts"]
|
||||||
|
|
||||||
|
|||||||
@@ -385,17 +385,19 @@ def test_processing_correctness(
|
|||||||
num_batches: int,
|
num_batches: int,
|
||||||
simplify_rate: float,
|
simplify_rate: float,
|
||||||
):
|
):
|
||||||
if model_id == "google/gemma-3n-E2B-it":
|
|
||||||
pytest.skip("Fix later")
|
|
||||||
if model_id == "OpenGVLab/InternVL2-2B":
|
|
||||||
pytest.skip("Fix later")
|
|
||||||
if model_id == "openvla/openvla-7b":
|
if model_id == "openvla/openvla-7b":
|
||||||
pytest.skip(
|
pytest.skip(
|
||||||
"OpenVLA uses a custom vLLM processor because its HF remote "
|
"OpenVLA uses a custom vLLM processor because its HF remote "
|
||||||
"processor is incompatible with current Transformers."
|
"processor is incompatible with current Transformers."
|
||||||
)
|
)
|
||||||
if model_id == "jinaai/jina-reranker-m0":
|
if model_id == "jinaai/jina-reranker-m0":
|
||||||
pytest.skip("Fix later")
|
pytest.skip(
|
||||||
|
"JinaVL reverses the multi-modal item order inside "
|
||||||
|
"_apply_hf_processor_main to match its document-before-query "
|
||||||
|
"score template, which breaks the positional correspondence "
|
||||||
|
"between multi-modal kwargs and hashes assumed by the "
|
||||||
|
"processing cache. Needs a fix that keeps both consistent."
|
||||||
|
)
|
||||||
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
|
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
|
||||||
pytest.skip(
|
pytest.skip(
|
||||||
"Voxtral Realtime doesn't make use of any place-holder "
|
"Voxtral Realtime doesn't make use of any place-holder "
|
||||||
@@ -403,22 +405,12 @@ def test_processing_correctness(
|
|||||||
"correctness test as is. Let's revisit adapting this "
|
"correctness test as is. Let's revisit adapting this "
|
||||||
"test once more realtime models exist."
|
"test once more realtime models exist."
|
||||||
)
|
)
|
||||||
if model_id == "CohereLabs/cohere-transcribe-03-2026":
|
|
||||||
pytest.skip("Fix later")
|
|
||||||
if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"):
|
if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"):
|
||||||
pytest.skip(
|
pytest.skip(
|
||||||
"MOSS-Audio uses a custom processor that dynamically expands "
|
"MOSS-Audio uses a custom processor that dynamically expands "
|
||||||
"audio placeholders from processed audio lengths. Its vLLM "
|
"audio placeholders from processed audio lengths. Its vLLM "
|
||||||
"processor paths are covered by test_moss_audio.py."
|
"processor paths are covered by test_moss_audio.py."
|
||||||
)
|
)
|
||||||
# TODO: Remove when transformers 5.15.0 is released, which contains
|
|
||||||
# https://github.com/huggingface/transformers/pull/47483.
|
|
||||||
if model_id == "microsoft/VibeVoice-ASR-HF":
|
|
||||||
pytest.skip(
|
|
||||||
"VibeVoice ASR requires audio as a positional argument and hence "
|
|
||||||
"cannot pass the processing correctness test as is. Its generation "
|
|
||||||
"is covered by test_transformers_audio.py."
|
|
||||||
)
|
|
||||||
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
|
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
|
||||||
pytest.skip(
|
pytest.skip(
|
||||||
"LLaVA-OneVision-2 video processing routes frames through custom "
|
"LLaVA-OneVision-2 video processing routes frames through custom "
|
||||||
|
|||||||
@@ -202,7 +202,7 @@ def test_validate_relative_bare_path_blocked():
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps)
|
# Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps)
|
||||||
#
|
#
|
||||||
# Non-codec videos reach _call_hf_processor as a ``(frames_ndarray, metadata)``
|
# Non-codec videos reach _apply_hf_processor_main as a ``(frames_ndarray, metadata)``
|
||||||
# tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real
|
# tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real
|
||||||
# ``video_url`` inputs, or by the dummy-inputs builder during profiling
|
# ``video_url`` inputs, or by the dummy-inputs builder during profiling
|
||||||
# (``video_needs_metadata=True``). The helper materialises PIL frames and
|
# (``video_needs_metadata=True``). The helper materialises PIL frames and
|
||||||
|
|||||||
@@ -103,7 +103,7 @@ def test_processor_multi_video(
|
|||||||
"""Verify that multi-video processing produces correct placeholders.
|
"""Verify that multi-video processing produces correct placeholders.
|
||||||
|
|
||||||
This exercises the token-level replacement path in
|
This exercises the token-level replacement path in
|
||||||
``_call_hf_processor`` which avoids the quadratic text-level
|
``_apply_hf_processor_main`` which avoids the quadratic text-level
|
||||||
prompt expansion.
|
prompt expansion.
|
||||||
"""
|
"""
|
||||||
ctx = build_model_context(
|
ctx = build_model_context(
|
||||||
@@ -153,7 +153,7 @@ def test_processor_multi_video_list_kwargs(
|
|||||||
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
|
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
|
||||||
crash.
|
crash.
|
||||||
|
|
||||||
Before the fix, ``_call_hf_processor`` copied the whole kwargs to every
|
Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
|
||||||
video without slicing, so ``_get_video_second_idx`` received the list
|
video without slicing, so ``_get_video_second_idx`` received the list
|
||||||
where a scalar was expected and raised ``TypeError``.
|
where a scalar was expected and raised ``TypeError``.
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -377,35 +377,41 @@ class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):
|
|||||||
class AudioFlamingo3MultiModalProcessor(
|
class AudioFlamingo3MultiModalProcessor(
|
||||||
BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo]
|
BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, Any],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
) -> BatchFeature:
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
processor_mm_data = dict(mm_data)
|
mm_data = dict(mm_data)
|
||||||
audios = processor_mm_data.pop("audios", None)
|
audios = mm_data.pop("audios", None)
|
||||||
if audios is not None:
|
if audios is not None:
|
||||||
processor_mm_data["audio"] = audios
|
mm_data["audio"] = audios
|
||||||
|
|
||||||
outputs = super()._call_hf_processor(
|
return mm_data, hf_processor_mm_kwargs
|
||||||
prompt=prompt,
|
|
||||||
mm_data=processor_mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
if "input_features_mask" in outputs:
|
def _postprocess_hf_mm_data(
|
||||||
outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if "input_features_mask" in processed_data:
|
||||||
|
processed_data["feature_attention_mask"] = processed_data.pop(
|
||||||
|
"input_features_mask"
|
||||||
|
)
|
||||||
|
|
||||||
audio_data = processor_mm_data.get("audio")
|
audio_data = mm_data.get("audio")
|
||||||
if audio_data is None:
|
if audio_data is None:
|
||||||
return outputs
|
return processed_data
|
||||||
|
|
||||||
audio_list = audio_data if isinstance(audio_data, list) else [audio_data]
|
audio_list = audio_data if isinstance(audio_data, list) else [audio_data]
|
||||||
if len(audio_list) == 0:
|
if len(audio_list) == 0:
|
||||||
return outputs
|
return processed_data
|
||||||
|
|
||||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
feature_extractor = processor.feature_extractor
|
feature_extractor = processor.feature_extractor
|
||||||
sampling_rate = feature_extractor.sampling_rate
|
sampling_rate = feature_extractor.sampling_rate
|
||||||
chunk_length = feature_extractor.chunk_length
|
chunk_length = feature_extractor.chunk_length
|
||||||
@@ -423,8 +429,9 @@ class AudioFlamingo3MultiModalProcessor(
|
|||||||
n_win = max_windows
|
n_win = max_windows
|
||||||
chunk_counts.append(n_win)
|
chunk_counts.append(n_win)
|
||||||
|
|
||||||
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
|
processed_data["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
|
||||||
return outputs
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -467,24 +467,6 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
|
class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
|
||||||
def _call_hf_processor(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
if not mm_data:
|
|
||||||
# HF processor always adds placeholders even when there's no image
|
|
||||||
tokenizer = self.info.get_tokenizer()
|
|
||||||
prompt_ids = tokenizer.encode(prompt)
|
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
|
||||||
return super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
hf_inputs: BatchFeature,
|
hf_inputs: BatchFeature,
|
||||||
|
|||||||
@@ -135,34 +135,16 @@ class ChameleonDummyInputsBuilder(BaseDummyInputsBuilder[ChameleonProcessingInfo
|
|||||||
|
|
||||||
|
|
||||||
class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingInfo]):
|
class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
# ChameleonProcessor requires text corresponding to the images
|
||||||
prompt: str,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
if not mm_data:
|
|
||||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
|
||||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
|
||||||
return super()._call_hf_processor(
|
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _apply_hf_processor_tokens_only(
|
|
||||||
self,
|
|
||||||
prompt_tokens: list[int],
|
|
||||||
) -> list[int]:
|
|
||||||
# HF processor adds sep token for chat mode
|
# HF processor adds sep token for chat mode
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
vocab = tokenizer.get_vocab()
|
sep_token_id = tokenizer.get_vocab()[tokenizer.sep_token] # type: ignore
|
||||||
|
|
||||||
sep_token_id = vocab[tokenizer.sep_token] # type: ignore
|
return [*prompt, sep_token_id]
|
||||||
|
|
||||||
return prompt_tokens + [sep_token_id]
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ import torch
|
|||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
from einops import rearrange
|
from einops import rearrange
|
||||||
from transformers import BatchFeature
|
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
@@ -492,14 +491,6 @@ class CheersDummyInputsBuilder(BaseDummyInputsBuilder[CheersProcessingInfo]):
|
|||||||
class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):
|
class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):
|
||||||
"""Multimodal processor for Cheers model."""
|
"""Multimodal processor for Cheers model."""
|
||||||
|
|
||||||
def _call_hf_processor(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
mm_items: MultiModalDataItems,
|
mm_items: MultiModalDataItems,
|
||||||
|
|||||||
@@ -222,24 +222,23 @@ class Cohere2VisionDummyInputsBuilder(
|
|||||||
class Cohere2VisionMultiModalProcessor(
|
class Cohere2VisionMultiModalProcessor(
|
||||||
BaseMultiModalProcessor[Cohere2VisionProcessingInfo]
|
BaseMultiModalProcessor[Cohere2VisionProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
processed_outputs = super()._call_hf_processor(
|
if not mm_data:
|
||||||
prompt,
|
return processed_data
|
||||||
mm_data,
|
|
||||||
mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Ensure num_patches is available for proper tensor splitting
|
|
||||||
if (
|
if (
|
||||||
"num_patches" not in processed_outputs
|
"num_patches" not in processed_data
|
||||||
and (images := mm_data.get("images")) is not None
|
and (images := mm_data.get("images")) is not None
|
||||||
):
|
):
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
|
||||||
# Fallback calculation if HF processor didn't provide num_patches
|
# Fallback calculation if HF processor didn't provide num_patches
|
||||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||||
@@ -250,13 +249,13 @@ class Cohere2VisionMultiModalProcessor(
|
|||||||
image_width=parsed_images.get_image_size(i).width,
|
image_width=parsed_images.get_image_size(i).width,
|
||||||
image_height=parsed_images.get_image_size(i).height,
|
image_height=parsed_images.get_image_size(i).height,
|
||||||
processor=hf_processor,
|
processor=hf_processor,
|
||||||
mm_kwargs=mm_kwargs,
|
mm_kwargs=hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
for i in range(len(parsed_images))
|
for i in range(len(parsed_images))
|
||||||
]
|
]
|
||||||
processed_outputs["num_patches"] = torch.tensor(num_patches)
|
processed_data["num_patches"] = torch.tensor(num_patches)
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from typing import Any, ClassVar
|
|||||||
import torch
|
import torch
|
||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
from torch import nn
|
from torch import nn
|
||||||
from transformers import PretrainedConfig
|
from transformers import BatchFeature, PretrainedConfig
|
||||||
|
|
||||||
from vllm.compilation.decorators import support_torch_compile
|
from vllm.compilation.decorators import support_torch_compile
|
||||||
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
|
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
|
||||||
@@ -45,9 +45,12 @@ from vllm.multimodal.processing import (
|
|||||||
BaseDummyInputsBuilder,
|
BaseDummyInputsBuilder,
|
||||||
BaseProcessingInfo,
|
BaseProcessingInfo,
|
||||||
EncDecMultiModalProcessor,
|
EncDecMultiModalProcessor,
|
||||||
|
ProcessorInputs,
|
||||||
PromptReplacement,
|
PromptReplacement,
|
||||||
PromptUpdate,
|
PromptUpdate,
|
||||||
|
TimingContext,
|
||||||
)
|
)
|
||||||
|
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
|
||||||
from vllm.renderers import TokenizeParams
|
from vllm.renderers import TokenizeParams
|
||||||
from vllm.tokenizers import cached_tokenizer_from_config
|
from vllm.tokenizers import cached_tokenizer_from_config
|
||||||
from vllm.transformers_utils.processors.cohere_asr import (
|
from vllm.transformers_utils.processors.cohere_asr import (
|
||||||
@@ -1941,27 +1944,51 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
|
|||||||
) -> list[int]:
|
) -> list[int]:
|
||||||
return [0]
|
return [0]
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
):
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
if mm_data:
|
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
|
||||||
mm_data = dict(audio=dict(mm_data).pop("audios"))
|
mm_data = dict(mm_data)
|
||||||
mm_kwargs = dict(
|
mm_data["audio"] = mm_data.pop("audios")
|
||||||
**mm_kwargs,
|
|
||||||
sampling_rate=feature_extractor.sampling_rate,
|
hf_processor_mm_kwargs = dict(
|
||||||
)
|
**hf_processor_mm_kwargs,
|
||||||
processed_outputs = super()._call_hf_processor(
|
sampling_rate=feature_extractor.sampling_rate,
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
)
|
||||||
if "labels" in processed_outputs:
|
|
||||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
return mm_data, hf_processor_mm_kwargs
|
||||||
return processed_outputs
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if "labels" in processed_data:
|
||||||
|
processed_data["input_ids"] = processed_data.pop("labels")
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
|
def _cached_apply_hf_processor(
|
||||||
|
self,
|
||||||
|
inputs: ProcessorInputs,
|
||||||
|
timing_ctx: TimingContext,
|
||||||
|
) -> MultiModalProcessingInfo:
|
||||||
|
# Dithering injects noise into the extracted features, so the
|
||||||
|
# feature extractor is not a pure function of its input. Since the
|
||||||
|
# processing cache assumes that processor outputs are invariant
|
||||||
|
# across calls, bypass the cache when dithering is active.
|
||||||
|
preproc = self.info.get_hf_config().preprocessor
|
||||||
|
if preproc.get("dither", 1e-05) > 0:
|
||||||
|
return self._apply_hf_processor(inputs, timing_ctx)
|
||||||
|
|
||||||
|
return super()._cached_apply_hf_processor(inputs, timing_ctx)
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -154,16 +154,25 @@ class ColModernVBertDummyInputsBuilder(
|
|||||||
class ColModernVBertMultiModalProcessor(
|
class ColModernVBertMultiModalProcessor(
|
||||||
BaseMultiModalProcessor[ColModernVBertProcessingInfo],
|
BaseMultiModalProcessor[ColModernVBertProcessingInfo],
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
assert isinstance(tokenizer, HfTokenizer)
|
assert isinstance(tokenizer, HfTokenizer)
|
||||||
text_encoding = tokenizer(
|
text_encoding = tokenizer(
|
||||||
prompt,
|
prompt_text,
|
||||||
return_tensors="pt",
|
return_tensors="pt",
|
||||||
)
|
)
|
||||||
result = BatchFeature(data=dict(text_encoding))
|
result = BatchFeature(data=dict(text_encoding))
|
||||||
@@ -183,7 +192,9 @@ class ColModernVBertMultiModalProcessor(
|
|||||||
)
|
)
|
||||||
result.update(image_outputs)
|
result.update(image_outputs)
|
||||||
|
|
||||||
return result
|
processed_data = result
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
|
|||||||
from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead
|
from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead
|
||||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
|
from vllm.multimodal.parse import MultiModalDataItems
|
||||||
|
|
||||||
from .interfaces import SupportsLateInteraction
|
from .interfaces import SupportsLateInteraction
|
||||||
from .interfaces_base import default_pooling_type
|
from .interfaces_base import default_pooling_type
|
||||||
@@ -59,24 +60,31 @@ class ColPaliProcessingInfo(PaliGemmaProcessingInfo):
|
|||||||
class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
|
class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
|
||||||
"""Multimodal processor for ColPali."""
|
"""Multimodal processor for ColPali."""
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
if mm_data:
|
valid_mm_items = mm_items.select(
|
||||||
# The ColPali tokenizer_config.json ships with a small default
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
# max_length (50) that truncates the 1024 image tokens inserted
|
)
|
||||||
# by PaliGemmaProcessor, causing a token-count mismatch.
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
# vLLM enforces its own max_model_len, so we disable HF
|
|
||||||
# truncation to keep all image + text tokens intact.
|
if not mm_data:
|
||||||
return self.info.ctx.call_hf_processor(
|
return BatchFeature(dict(passthrough_data))
|
||||||
self.info.get_hf_processor(**mm_kwargs),
|
|
||||||
dict(text=prompt, **mm_data),
|
# The ColPali tokenizer_config.json ships with a small default
|
||||||
dict(**mm_kwargs, truncation=False),
|
# max_length (50) that truncates the 1024 image tokens inserted
|
||||||
)
|
# by PaliGemmaProcessor, causing a token-count mismatch.
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
# vLLM enforces its own max_model_len, so we disable HF
|
||||||
|
# truncation to keep all image + text tokens intact.
|
||||||
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
|
mm_data,
|
||||||
|
dict(**hf_processor_mm_kwargs, truncation=False),
|
||||||
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
|
|
||||||
@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
|
@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
|
||||||
|
|||||||
@@ -287,27 +287,34 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing
|
|||||||
class DeepseekOCRMultiModalProcessor(
|
class DeepseekOCRMultiModalProcessor(
|
||||||
BaseMultiModalProcessor[DeepseekOCRProcessingInfo]
|
BaseMultiModalProcessor[DeepseekOCRProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
if mm_data:
|
if mm_data:
|
||||||
processed_outputs = self.info.ctx.call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
self.info.get_hf_processor(**mm_kwargs),
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
dict(prompt=prompt, **mm_data),
|
dict(prompt=prompt_text, **mm_data),
|
||||||
mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
assert isinstance(tokenizer, HfTokenizer)
|
assert isinstance(tokenizer, HfTokenizer)
|
||||||
processed_outputs = tokenizer(
|
processed_data = tokenizer(
|
||||||
prompt, add_special_tokens=True, return_tensors="pt"
|
prompt_text, add_special_tokens=True, return_tensors="pt"
|
||||||
)
|
)
|
||||||
|
|
||||||
return processed_outputs
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -158,27 +158,35 @@ class DeepseekOCR2DummyInputsBuilder(
|
|||||||
class DeepseekOCR2MultiModalProcessor(
|
class DeepseekOCR2MultiModalProcessor(
|
||||||
BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo]
|
BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
if mm_data:
|
if mm_data:
|
||||||
processed_outputs = self.info.ctx.call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
self.info.get_hf_processor(**mm_kwargs),
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
dict(prompt=prompt, **mm_data),
|
dict(prompt=prompt_text, **mm_data),
|
||||||
mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
assert isinstance(tokenizer, HfTokenizer)
|
assert isinstance(tokenizer, HfTokenizer)
|
||||||
processed_outputs = tokenizer(
|
processed_data = tokenizer(
|
||||||
prompt, add_special_tokens=True, return_tensors="pt"
|
prompt_text, add_special_tokens=True, return_tensors="pt"
|
||||||
)
|
)
|
||||||
|
|
||||||
return processed_outputs
|
processed_data.update(passthrough_data)
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -43,7 +43,6 @@ from vllm.multimodal.processing.processor import (
|
|||||||
)
|
)
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.tokenizers import cached_tokenizer_from_config
|
from vllm.tokenizers import cached_tokenizer_from_config
|
||||||
from vllm.tokenizers.hf import HfTokenizer
|
|
||||||
from vllm.transformers_utils.configs.deepseek_vl2 import (
|
from vllm.transformers_utils.configs.deepseek_vl2 import (
|
||||||
DeepseekVLV2Config,
|
DeepseekVLV2Config,
|
||||||
MlpProjectorConfig,
|
MlpProjectorConfig,
|
||||||
@@ -236,28 +235,23 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
|
|||||||
class DeepseekVL2MultiModalProcessor(
|
class DeepseekVL2MultiModalProcessor(
|
||||||
BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]
|
BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
if not mm_data:
|
if not mm_data:
|
||||||
tokenizer = self.info.get_tokenizer()
|
return processed_data
|
||||||
assert isinstance(tokenizer, HfTokenizer)
|
|
||||||
return tokenizer(prompt, add_special_tokens=True, return_tensors="pt")
|
|
||||||
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
processed_data["num_patches"] = (
|
||||||
prompt=prompt,
|
processed_data["images_spatial_crop"].prod(-1) + 1
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
processed_outputs["num_patches"] = (
|
return processed_data
|
||||||
processed_outputs["images_spatial_crop"].prod(-1) + 1
|
|
||||||
)
|
|
||||||
|
|
||||||
return processed_outputs
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
@@ -315,7 +309,7 @@ class DeepseekVL2MultiModalProcessor(
|
|||||||
self,
|
self,
|
||||||
inputs: ProcessorInputs,
|
inputs: ProcessorInputs,
|
||||||
timing_ctx: TimingContext,
|
timing_ctx: TimingContext,
|
||||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
) -> MultiModalProcessingInfo:
|
||||||
# The processor logic is different for len(images) <= 2 vs > 2
|
# The processor logic is different for len(images) <= 2 vs > 2
|
||||||
# Since the processing cache assumes that the processor output is
|
# Since the processing cache assumes that the processor output is
|
||||||
# invariant of how many images are passed per prompt, we only
|
# invariant of how many images are passed per prompt, we only
|
||||||
|
|||||||
@@ -1110,21 +1110,20 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
|||||||
pixel_values = pixel_values.to(hf_config.dtype)
|
pixel_values = pixel_values.to(hf_config.dtype)
|
||||||
return pixel_values
|
return pixel_values
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
# when the prompt is not empty but the multimodal data is empty,
|
valid_mm_items = mm_items.select(
|
||||||
# directly invoke the tokenizer.
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
if "images" not in mm_data and "videos" not in mm_data and prompt != "":
|
)
|
||||||
tokenizer = self.info.get_tokenizer()
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
prompt_ids = tokenizer.encode(prompt)
|
|
||||||
tokenizer_output = BatchFeature(
|
if "images" not in mm_data and "videos" not in mm_data:
|
||||||
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
return BatchFeature(dict(passthrough_data))
|
||||||
)
|
|
||||||
return tokenizer_output
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
if "images" not in mm_data:
|
if "images" not in mm_data:
|
||||||
mm_data["images"] = []
|
mm_data["images"] = []
|
||||||
@@ -1132,7 +1131,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
|||||||
mm_data["videos"] = []
|
mm_data["videos"] = []
|
||||||
|
|
||||||
# Check if HF processor supports video metadata
|
# Check if HF processor supports video metadata
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
supports_video_metadata = getattr(
|
supports_video_metadata = getattr(
|
||||||
hf_processor, "supports_video_metadata", False
|
hf_processor, "supports_video_metadata", False
|
||||||
)
|
)
|
||||||
@@ -1149,8 +1148,10 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
|||||||
|
|
||||||
processor_output = self.info.ctx.call_hf_processor(
|
processor_output = self.info.ctx.call_hf_processor(
|
||||||
hf_processor,
|
hf_processor,
|
||||||
dict(text=[prompt], images=mm_data["images"], videos=mm_data["videos"]),
|
dict(
|
||||||
mm_kwargs,
|
text=[prompt_text], images=mm_data["images"], videos=mm_data["videos"]
|
||||||
|
),
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Divide the processor_output into two modalities: image and video.
|
# Divide the processor_output into two modalities: image and video.
|
||||||
@@ -1158,7 +1159,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
|||||||
pixel_values = processor_output["images"]
|
pixel_values = processor_output["images"]
|
||||||
if pixel_values is not None:
|
if pixel_values is not None:
|
||||||
processor_output["images"] = self._pixel_values_norm(
|
processor_output["images"] = self._pixel_values_norm(
|
||||||
pixel_values, mm_kwargs
|
pixel_values, hf_processor_mm_kwargs
|
||||||
)
|
)
|
||||||
for key in list(processor_output.keys()):
|
for key in list(processor_output.keys()):
|
||||||
if processor_output[key] is None:
|
if processor_output[key] is None:
|
||||||
@@ -1184,7 +1185,9 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
|||||||
]
|
]
|
||||||
del processor_output["images"]
|
del processor_output["images"]
|
||||||
|
|
||||||
return processor_output
|
processed_data = processor_output
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -228,27 +228,36 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing
|
|||||||
class FireRedASR2MultiModalProcessor(
|
class FireRedASR2MultiModalProcessor(
|
||||||
BaseMultiModalProcessor[FireRedASR2ProcessingInfo]
|
BaseMultiModalProcessor[FireRedASR2ProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
) -> BatchFeature:
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
if mm_data:
|
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
|
||||||
mm_data = dict(audio=mm_data.pop("audios"))
|
mm_data = dict(mm_data)
|
||||||
mm_kwargs = dict(
|
mm_data["audio"] = mm_data.pop("audios")
|
||||||
**mm_kwargs,
|
|
||||||
sampling_rate=feature_extractor.sampling_rate,
|
hf_processor_mm_kwargs = dict(
|
||||||
)
|
**hf_processor_mm_kwargs,
|
||||||
processed_outputs = super()._call_hf_processor(
|
sampling_rate=feature_extractor.sampling_rate,
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
)
|
||||||
if "labels" in processed_outputs:
|
|
||||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
return mm_data, hf_processor_mm_kwargs
|
||||||
return processed_outputs
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if "labels" in processed_data:
|
||||||
|
processed_data["input_ids"] = processed_data.pop("labels")
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -451,27 +451,36 @@ class FireRedLIDMultiModalProcessor(
|
|||||||
# Dummy encoder prompt for profiling (encoder only processes audio).
|
# Dummy encoder prompt for profiling (encoder only processes audio).
|
||||||
return [0]
|
return [0]
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
) -> BatchFeature:
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
if mm_data:
|
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
|
||||||
mm_data = dict(audio=mm_data.pop("audios"))
|
mm_data = dict(mm_data)
|
||||||
mm_kwargs = dict(
|
mm_data["audio"] = mm_data.pop("audios")
|
||||||
**mm_kwargs,
|
|
||||||
sampling_rate=feature_extractor.sampling_rate,
|
hf_processor_mm_kwargs = dict(
|
||||||
)
|
**hf_processor_mm_kwargs,
|
||||||
processed_outputs = super()._call_hf_processor(
|
sampling_rate=feature_extractor.sampling_rate,
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
)
|
||||||
if "labels" in processed_outputs:
|
|
||||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
return mm_data, hf_processor_mm_kwargs
|
||||||
return processed_outputs
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if "labels" in processed_data:
|
||||||
|
processed_data["input_ids"] = processed_data.pop("labels")
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -44,9 +44,12 @@ from vllm.multimodal.processing import (
|
|||||||
BaseDummyInputsBuilder,
|
BaseDummyInputsBuilder,
|
||||||
BaseMultiModalProcessor,
|
BaseMultiModalProcessor,
|
||||||
BaseProcessingInfo,
|
BaseProcessingInfo,
|
||||||
|
ProcessorInputs,
|
||||||
PromptReplacement,
|
PromptReplacement,
|
||||||
PromptUpdate,
|
PromptUpdate,
|
||||||
|
TimingContext,
|
||||||
)
|
)
|
||||||
|
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
|
||||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||||
from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor
|
from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor
|
||||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||||
@@ -747,27 +750,50 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
|
class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
) -> BatchFeature:
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
if mm_data:
|
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
|
||||||
mm_data = dict(audio=mm_data.pop("audios"))
|
mm_data = dict(mm_data)
|
||||||
mm_kwargs = dict(
|
mm_data["audio"] = mm_data.pop("audios")
|
||||||
**mm_kwargs,
|
|
||||||
sampling_rate=feature_extractor.sampling_rate,
|
hf_processor_mm_kwargs = dict(
|
||||||
)
|
**hf_processor_mm_kwargs,
|
||||||
processed_outputs = super()._call_hf_processor(
|
sampling_rate=feature_extractor.sampling_rate,
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
)
|
||||||
if "labels" in processed_outputs:
|
|
||||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
return mm_data, hf_processor_mm_kwargs
|
||||||
return processed_outputs
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if "labels" in processed_data:
|
||||||
|
processed_data["input_ids"] = processed_data.pop("labels")
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
|
def _cached_apply_hf_processor(
|
||||||
|
self,
|
||||||
|
inputs: ProcessorInputs,
|
||||||
|
timing_ctx: TimingContext,
|
||||||
|
) -> MultiModalProcessingInfo:
|
||||||
|
# Dithering injects noise into the extracted features, so the
|
||||||
|
# feature extractor is not a pure function of its input. Since the
|
||||||
|
# processing cache assumes that processor outputs are invariant
|
||||||
|
# across calls, bypass the cache when dithering is active.
|
||||||
|
if self.info.get_feature_extractor().dither > 0:
|
||||||
|
return self._apply_hf_processor(inputs, timing_ctx)
|
||||||
|
|
||||||
|
return super()._cached_apply_hf_processor(inputs, timing_ctx)
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -612,20 +612,31 @@ class FunAudioChatDummyInputsBuilder(
|
|||||||
class FunAudioChatMultiModalProcessor(
|
class FunAudioChatMultiModalProcessor(
|
||||||
BaseMultiModalProcessor[FunAudioChatProcessingInfo]
|
BaseMultiModalProcessor[FunAudioChatProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
input_ids = torch.tensor([tokenizer.encode(prompt)])
|
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
|
||||||
|
|
||||||
audios = mm_data.get("audios", [])
|
audios = mm_data.get("audios", [])
|
||||||
if not audios:
|
if not audios:
|
||||||
return BatchFeature({"input_ids": input_ids})
|
processed_data = BatchFeature({"input_ids": input_ids})
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||||
sr = int(feature_extractor.sampling_rate)
|
sr = int(feature_extractor.sampling_rate)
|
||||||
min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400)
|
min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400)
|
||||||
|
|
||||||
@@ -676,7 +687,9 @@ class FunAudioChatMultiModalProcessor(
|
|||||||
"feature_exist_mask": torch.ones((len(wavs),), dtype=torch.bool),
|
"feature_exist_mask": torch.ones((len(wavs),), dtype=torch.bool),
|
||||||
}
|
}
|
||||||
|
|
||||||
return BatchFeature({"input_ids": input_ids, **mm_inputs})
|
processed_data = BatchFeature({"input_ids": input_ids, **mm_inputs})
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -261,39 +261,38 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
processed_outputs = super()._call_hf_processor(
|
if not mm_data:
|
||||||
prompt,
|
return processed_data
|
||||||
mm_data,
|
|
||||||
mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
# HF processor pops the `num_crops` kwarg, which is needed by vLLM
|
|
||||||
if (images := mm_data.get("images")) is not None:
|
if (images := mm_data.get("images")) is not None:
|
||||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||||
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
||||||
image_sizes = [
|
image_sizes = [
|
||||||
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
||||||
]
|
]
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
|
||||||
num_crops = [
|
num_crops = [
|
||||||
self.info.get_num_crops(
|
self.info.get_num_crops(
|
||||||
image_width=size.width,
|
image_width=size.width,
|
||||||
image_height=size.height,
|
image_height=size.height,
|
||||||
processor=hf_processor,
|
processor=hf_processor,
|
||||||
mm_kwargs=mm_kwargs,
|
mm_kwargs=hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
for size in image_sizes
|
for size in image_sizes
|
||||||
]
|
]
|
||||||
processed_outputs["num_patches"] = torch.tensor(num_crops) + 1
|
processed_data["num_patches"] = torch.tensor(num_crops) + 1
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import inspect
|
||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from typing import Annotated, Any, Literal
|
from typing import Annotated, Any, Literal
|
||||||
|
|
||||||
@@ -257,40 +258,69 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]):
|
class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
) -> BatchFeature:
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
# HF Transformers audio processor no longer accepts `audios` key.
|
mm_data = dict(mm_data)
|
||||||
# We pop `audios` and replace it with `audio` key to suppress
|
|
||||||
# the warning.
|
|
||||||
if "audios" in mm_data:
|
if "audios" in mm_data:
|
||||||
mm_data["audio"] = mm_data.pop("audios")
|
mm_data["audio"] = mm_data.pop("audios")
|
||||||
processed_outputs = super()._call_hf_processor(
|
|
||||||
prompt,
|
|
||||||
mm_data,
|
|
||||||
mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
if "input_features" in processed_outputs:
|
return mm_data, hf_processor_mm_kwargs
|
||||||
# Padding enables audio_tower to run in batched mode
|
|
||||||
processed_outputs["input_features_padded"] = processed_outputs[
|
def _postprocess_hf_mm_data(
|
||||||
"input_features"
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if "input_features" in processed_data:
|
||||||
|
# The feature extractor pads every clip to the longest one in
|
||||||
|
# the batch, so each clip's frame count depends on the other
|
||||||
|
# clips it was processed with. Trim each clip to the number of
|
||||||
|
# frames it would have if processed alone so that each item's
|
||||||
|
# output (and thus its cache entry) is independent of the other
|
||||||
|
# items in the batch. `batch_audio_features` re-pads the
|
||||||
|
# per-item features so audio_tower can still run batched.
|
||||||
|
num_frames = [
|
||||||
|
self._get_num_audio_frames_alone(len(audio))
|
||||||
|
for audio in mm_data["audio"]
|
||||||
|
]
|
||||||
|
processed_data["input_features_padded"] = [
|
||||||
|
features[:n]
|
||||||
|
for features, n in zip(processed_data.pop("input_features"), num_frames)
|
||||||
|
]
|
||||||
|
processed_data["input_features_mask"] = [
|
||||||
|
mask[:n]
|
||||||
|
for mask, n in zip(processed_data["input_features_mask"], num_frames)
|
||||||
]
|
]
|
||||||
|
|
||||||
# Unpad features here since we need the output of each item to be
|
return processed_data
|
||||||
# independent of other items for the cache to work correctly
|
|
||||||
unpadded_features = [
|
def _get_num_audio_frames_alone(self, num_samples: int) -> int:
|
||||||
f[mask]
|
"""
|
||||||
for f, mask in zip(
|
Get the number of mel frames that the feature extractor produces
|
||||||
processed_outputs["input_features"],
|
for a single audio clip of the given length, independent of batch
|
||||||
processed_outputs["input_features_mask"],
|
padding.
|
||||||
)
|
"""
|
||||||
]
|
feature_extractor = self.info.get_feature_extractor()
|
||||||
processed_outputs["input_features"] = unpadded_features
|
call_params = inspect.signature(type(feature_extractor).__call__).parameters
|
||||||
return processed_outputs
|
max_length = call_params["max_length"].default
|
||||||
|
pad_to_multiple_of = call_params["pad_to_multiple_of"].default
|
||||||
|
|
||||||
|
num_samples = min(num_samples, max_length)
|
||||||
|
remainder = num_samples % pad_to_multiple_of
|
||||||
|
if remainder:
|
||||||
|
num_samples += pad_to_multiple_of - remainder
|
||||||
|
|
||||||
|
frame_size = feature_extractor.frame_length + 1
|
||||||
|
num_frames = (num_samples - frame_size) // feature_extractor.hop_length + 1
|
||||||
|
|
||||||
|
return max(num_frames, 0)
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -560,13 +560,22 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
|
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||||
val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
|
val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
|
||||||
|
|
||||||
if val is not None and val not in _SUPPORTED_SOFT_TOKENS:
|
if val is not None and val not in _SUPPORTED_SOFT_TOKENS:
|
||||||
@@ -611,15 +620,15 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
|||||||
timestamps = [idx / fps for idx in frame_indices]
|
timestamps = [idx / fps for idx in frame_indices]
|
||||||
|
|
||||||
# Process frames as images with max_soft_tokens=70
|
# Process frames as images with max_soft_tokens=70
|
||||||
video_mm_kwargs = dict(mm_kwargs)
|
video_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||||
video_mm_kwargs["max_soft_tokens"] = _VIDEO_MAX_SOFT_TOKENS
|
video_mm_kwargs["max_soft_tokens"] = _VIDEO_MAX_SOFT_TOKENS
|
||||||
|
|
||||||
dummy_prompt = ("\t" + processor.image_token) * len(frames)
|
dummy_prompt = ("\t" + processor.image_token) * len(frames)
|
||||||
|
|
||||||
frame_outputs = super()._call_hf_processor(
|
frame_outputs = self.info.ctx.call_hf_processor(
|
||||||
prompt=dummy_prompt,
|
self.info.get_hf_processor(**video_mm_kwargs),
|
||||||
mm_data={"images": frames},
|
dict(text=dummy_prompt, **{"images": frames}),
|
||||||
mm_kwargs=video_mm_kwargs,
|
video_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Remap HF key name
|
# Remap HF key name
|
||||||
@@ -660,7 +669,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
|||||||
# split-replace bug where replacement text (which itself
|
# split-replace bug where replacement text (which itself
|
||||||
# contains <|video|> tokens) collides with later splits.
|
# contains <|video|> tokens) collides with later splits.
|
||||||
vt = processor.video_token
|
vt = processor.video_token
|
||||||
parts = prompt.split(vt, len(video_replacements))
|
parts = prompt_text.split(vt, len(video_replacements))
|
||||||
|
|
||||||
# NOTE: len(parts) <= len(video_replacements) + 1
|
# NOTE: len(parts) <= len(video_replacements) + 1
|
||||||
parts_with_repl: list[str] = []
|
parts_with_repl: list[str] = []
|
||||||
@@ -668,7 +677,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
|||||||
parts_with_repl.extend([part, repl])
|
parts_with_repl.extend([part, repl])
|
||||||
parts_with_repl.extend(parts[len(video_replacements) :])
|
parts_with_repl.extend(parts[len(video_replacements) :])
|
||||||
|
|
||||||
prompt = "".join(parts_with_repl)
|
prompt_text = "".join(parts_with_repl)
|
||||||
|
|
||||||
video_outputs = {
|
video_outputs = {
|
||||||
"pixel_values_videos": torch.cat(all_video_pixel_values, dim=0),
|
"pixel_values_videos": torch.cat(all_video_pixel_values, dim=0),
|
||||||
@@ -712,45 +721,46 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
|||||||
# NOTE: This requires a corresponding type annotation on the
|
# NOTE: This requires a corresponding type annotation on the
|
||||||
# HF side (Gemma4ProcessorKwargs.images_kwargs) so that
|
# HF side (Gemma4ProcessorKwargs.images_kwargs) so that
|
||||||
# _merge_kwargs routes max_soft_tokens into images_kwargs.
|
# _merge_kwargs routes max_soft_tokens into images_kwargs.
|
||||||
patched_mm_kwargs = dict(mm_kwargs)
|
patched_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||||
if val is not None and is_top_level_max_soft_tokens:
|
if val is not None and is_top_level_max_soft_tokens:
|
||||||
patched_mm_kwargs["max_soft_tokens"] = val
|
patched_mm_kwargs["max_soft_tokens"] = val
|
||||||
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
prompt,
|
self.info.get_hf_processor(**patched_mm_kwargs),
|
||||||
mm_data,
|
dict(text=prompt_text, **mm_data),
|
||||||
patched_mm_kwargs,
|
patched_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
# HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'.
|
# HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'.
|
||||||
# Remap here to keep a single translation point.
|
# Remap here to keep a single translation point.
|
||||||
if "image_position_ids" in processed_outputs:
|
if "image_position_ids" in processed_data:
|
||||||
processed_outputs["pixel_position_ids"] = processed_outputs.pop(
|
processed_data["pixel_position_ids"] = processed_data.pop(
|
||||||
"image_position_ids"
|
"image_position_ids"
|
||||||
)
|
)
|
||||||
|
|
||||||
if "input_features" in processed_outputs:
|
if "input_features" in processed_data:
|
||||||
# Unpad per-item so each item's cache entry is
|
# Unpad per-item so each item's cache entry is
|
||||||
# self-contained. The batched() field config in
|
# self-contained. The batched() field config in
|
||||||
# _get_mm_fields_config stacks equal-length items and leaves
|
# _get_mm_fields_config stacks equal-length items and leaves
|
||||||
# the rest as a list; _process_audio_input re-pads that list
|
# the rest as a list; _process_audio_input re-pads that list
|
||||||
# to the batch's max length.
|
# to the batch's max length.
|
||||||
masks = processed_outputs["input_features_mask"]
|
masks = processed_data["input_features_mask"]
|
||||||
unpadded_features = [
|
unpadded_features = [
|
||||||
f[mask]
|
f[mask]
|
||||||
for f, mask in zip(
|
for f, mask in zip(
|
||||||
processed_outputs["input_features"],
|
processed_data["input_features"],
|
||||||
masks,
|
masks,
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
unpadded_masks = [mask[mask] for mask in masks]
|
unpadded_masks = [mask[mask] for mask in masks]
|
||||||
processed_outputs["input_features"] = unpadded_features
|
processed_data["input_features"] = unpadded_features
|
||||||
processed_outputs["input_features_padded"] = unpadded_features
|
processed_data["input_features_padded"] = unpadded_features
|
||||||
processed_outputs["input_features_mask"] = unpadded_masks
|
processed_data["input_features_mask"] = unpadded_masks
|
||||||
|
|
||||||
# Merge video outputs into the final result
|
processed_data.update(video_outputs)
|
||||||
combined_outputs = dict(processed_outputs, **video_outputs)
|
processed_data.update(passthrough_data)
|
||||||
return BatchFeature(combined_outputs)
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
@@ -815,7 +825,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
|||||||
image_size = images.get_image_size(item_idx)
|
image_size = images.get_image_size(item_idx)
|
||||||
# Resolve the effective max_soft_tokens by merging
|
# Resolve the effective max_soft_tokens by merging
|
||||||
# per-prompt kwargs with the config-level defaults,
|
# per-prompt kwargs with the config-level defaults,
|
||||||
# consistent with how _call_hf_processor resolves it.
|
# consistent with how _apply_hf_processor_main resolves it.
|
||||||
# Without this merge, a missing per-prompt override
|
# Without this merge, a missing per-prompt override
|
||||||
# would fall back to vision_cfg.default_output_length
|
# would fall back to vision_cfg.default_output_length
|
||||||
# instead of the config's mm_processor_kwargs default.
|
# instead of the config's mm_processor_kwargs default.
|
||||||
|
|||||||
@@ -1576,32 +1576,39 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
|
|
||||||
return prepared_data, prepared_kwargs
|
return prepared_data, prepared_kwargs
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
mm_data = dict(mm_data)
|
valid_mm_items = mm_items.select(
|
||||||
if not mm_data:
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
tokenizer = self.info.get_tokenizer()
|
)
|
||||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
|
||||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
|
mm_data = dict(mm_data)
|
||||||
|
|
||||||
|
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
|
||||||
use_direct_path = (
|
use_direct_path = (
|
||||||
not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
|
not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
|
||||||
)
|
)
|
||||||
if use_direct_path:
|
if use_direct_path:
|
||||||
prepared_data, prepared_kwargs = self._get_direct_path_inputs(
|
prepared_data, prepared_kwargs = self._get_direct_path_inputs(
|
||||||
mm_data, mm_kwargs
|
mm_data, hf_processor_mm_kwargs
|
||||||
)
|
)
|
||||||
return super()._call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
prompt=prompt,
|
self.info.get_hf_processor(**prepared_kwargs),
|
||||||
mm_data=prepared_data,
|
dict(text=prompt_text, **prepared_data),
|
||||||
mm_kwargs=prepared_kwargs,
|
prepared_kwargs,
|
||||||
)
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
if (
|
if (
|
||||||
"videos" in mm_data
|
"videos" in mm_data
|
||||||
@@ -1616,7 +1623,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
video_array, metadata = item
|
video_array, metadata = item
|
||||||
|
|
||||||
# don't update mm_kwargs inplace
|
# don't update mm_kwargs inplace
|
||||||
video_mm_kwargs = dict(**mm_kwargs)
|
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
|
||||||
video_mm_kwargs["do_sample_frames"] = metadata.get(
|
video_mm_kwargs["do_sample_frames"] = metadata.get(
|
||||||
"do_sample_frames", True
|
"do_sample_frames", True
|
||||||
)
|
)
|
||||||
@@ -1625,10 +1632,13 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
video_mm_data["videos"] = [[video_array]]
|
video_mm_data["videos"] = [[video_array]]
|
||||||
video_mm_data["video_metadata"] = [[_to_video_metadata(metadata)]]
|
video_mm_data["video_metadata"] = [[_to_video_metadata(metadata)]]
|
||||||
|
|
||||||
video_outputs = super()._call_hf_processor(
|
video_outputs = self.info.ctx.call_hf_processor(
|
||||||
prompt="<|begin_of_video|><|video|><|end_of_video|>",
|
self.info.get_hf_processor(**video_mm_kwargs),
|
||||||
mm_data=video_mm_data,
|
dict(
|
||||||
mm_kwargs=video_mm_kwargs,
|
text="<|begin_of_video|><|video|><|end_of_video|>",
|
||||||
|
**video_mm_data,
|
||||||
|
),
|
||||||
|
video_mm_kwargs,
|
||||||
)
|
)
|
||||||
input_ids = video_outputs.pop("input_ids")
|
input_ids = video_outputs.pop("input_ids")
|
||||||
if swap_video_frame_tokens:
|
if swap_video_frame_tokens:
|
||||||
@@ -1636,7 +1646,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
processor.video_token_id
|
processor.video_token_id
|
||||||
)
|
)
|
||||||
video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
|
video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
|
||||||
prompt = prompt.replace(
|
prompt_text = prompt_text.replace(
|
||||||
"<|begin_of_video|><|video|><|end_of_video|>",
|
"<|begin_of_video|><|video|><|end_of_video|>",
|
||||||
video_placeholder,
|
video_placeholder,
|
||||||
1,
|
1,
|
||||||
@@ -1652,20 +1662,20 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
video_outputs = dict()
|
video_outputs = dict()
|
||||||
swap_video_frame_tokens = False
|
swap_video_frame_tokens = False
|
||||||
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
prompt=prompt,
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
mm_data=mm_data,
|
dict(text=prompt_text, **mm_data),
|
||||||
mm_kwargs=mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
if swap_video_frame_tokens:
|
if swap_video_frame_tokens:
|
||||||
input_ids = processed_outputs["input_ids"]
|
input_ids = processed_data["input_ids"]
|
||||||
input_ids[input_ids == processor.video_token_id] = processor.image_token_id
|
input_ids[input_ids == processor.video_token_id] = processor.image_token_id
|
||||||
processed_outputs["input_ids"] = input_ids
|
processed_data["input_ids"] = input_ids
|
||||||
combined_outputs = dict(
|
|
||||||
processed_outputs,
|
processed_data.update(video_outputs)
|
||||||
**video_outputs,
|
processed_data.update(passthrough_data)
|
||||||
)
|
|
||||||
return BatchFeature(combined_outputs)
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -750,46 +750,40 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
|
|||||||
chunk_counts.append(min(n_chunks, max_windows))
|
chunk_counts.append(min(n_chunks, max_windows))
|
||||||
return chunk_counts
|
return chunk_counts
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_data: Mapping[str, object],
|
||||||
mm_data: dict[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, Any],
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
) -> BatchFeature:
|
mm_data = dict(mm_data)
|
||||||
# Normalize input: handle deprecated key and list conversion.
|
|
||||||
if "audios" in mm_data:
|
if "audios" in mm_data:
|
||||||
mm_data["audio"] = mm_data.pop("audios")
|
mm_data["audio"] = mm_data.pop("audios")
|
||||||
|
|
||||||
audio = mm_data.get("audio", [])
|
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||||
audio_list = [audio] if audio and not isinstance(audio, list) else audio
|
hf_processor_mm_kwargs = dict(
|
||||||
|
**hf_processor_mm_kwargs,
|
||||||
# Early return for text-only.
|
|
||||||
if not audio_list:
|
|
||||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
|
||||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
|
||||||
# Handle sampling_rate
|
|
||||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
|
||||||
mm_kwargs = dict(
|
|
||||||
**mm_kwargs,
|
|
||||||
sampling_rate=feature_extractor.sampling_rate,
|
sampling_rate=feature_extractor.sampling_rate,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Call parent method
|
return mm_data, hf_processor_mm_kwargs
|
||||||
outputs = super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Postprocess: rename mask and add chunk counts
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
# Handle different key names from different transformers versions
|
# Handle different key names from different transformers versions
|
||||||
if "input_features_mask" in outputs:
|
if "input_features_mask" in processed_data:
|
||||||
outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
|
processed_data["feature_attention_mask"] = processed_data.pop(
|
||||||
elif "input_features_mask" not in outputs and "input_features" in outputs:
|
"input_features_mask"
|
||||||
|
)
|
||||||
|
elif "input_features" in processed_data:
|
||||||
# If no mask is provided, create one from input_features
|
# If no mask is provided, create one from input_features
|
||||||
input_features = outputs["input_features"]
|
input_features = processed_data["input_features"]
|
||||||
if isinstance(input_features, torch.Tensor):
|
if isinstance(input_features, torch.Tensor):
|
||||||
# Create a mask of all ones matching the sequence length
|
# Create a mask of all ones matching the sequence length
|
||||||
mask = torch.ones(
|
mask = torch.ones(
|
||||||
@@ -797,18 +791,22 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
|
|||||||
input_features.shape[-1],
|
input_features.shape[-1],
|
||||||
dtype=torch.long,
|
dtype=torch.long,
|
||||||
)
|
)
|
||||||
outputs["feature_attention_mask"] = mask
|
processed_data["feature_attention_mask"] = mask
|
||||||
|
|
||||||
# Get processor for chunk counts calculation
|
audio = mm_data.get("audio", [])
|
||||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
audio_list = [audio] if audio and not isinstance(audio, list) else audio
|
||||||
|
if audio_list:
|
||||||
|
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
|
||||||
# Override chunk counts calculation with GLM-ASR specific logic
|
# Override chunk counts calculation with GLM-ASR specific logic
|
||||||
chunk_counts = self._calculate_chunk_counts(
|
chunk_counts = self._calculate_chunk_counts(
|
||||||
audio_list, processor.feature_extractor, processor
|
audio_list, processor.feature_extractor, processor
|
||||||
)
|
)
|
||||||
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
|
processed_data["chunk_counts"] = torch.tensor(
|
||||||
|
chunk_counts, dtype=torch.long
|
||||||
|
)
|
||||||
|
|
||||||
return outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -180,12 +180,14 @@ class GraniteSpeechMultiModalProcessor(
|
|||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
) -> BatchFeature:
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
mm_data = dict(mm_data)
|
mm_data = dict(mm_data)
|
||||||
audios = mm_data.pop("audios", [])
|
audios = mm_data.pop("audios", [])
|
||||||
|
|
||||||
@@ -193,21 +195,23 @@ class GraniteSpeechMultiModalProcessor(
|
|||||||
# GraniteSpeechFeatureExtractor accepts "audio"
|
# GraniteSpeechFeatureExtractor accepts "audio"
|
||||||
mm_data["audio"] = audios
|
mm_data["audio"] = audios
|
||||||
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
return mm_data, hf_processor_mm_kwargs
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
if "audio" in mm_data:
|
if "audio" in mm_data:
|
||||||
# Calculate the number of audio tokens per entry in the batch;
|
# Calculate the number of audio tokens per entry in the batch;
|
||||||
# This is used to split the batch back out after padding.
|
# This is used to split the batch back out after padding.
|
||||||
audio_token_index = self.info.get_hf_config().audio_token_index
|
audio_token_index = self.info.get_hf_config().audio_token_index
|
||||||
processed_outputs["audio_embed_sizes"] = (
|
processed_data["audio_embed_sizes"] = (
|
||||||
processed_outputs["input_ids"] == audio_token_index
|
processed_data["input_ids"] == audio_token_index
|
||||||
).sum(-1)
|
).sum(-1)
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
|
|
||||||
class GraniteSpeechDummyInputsBuilder(
|
class GraniteSpeechDummyInputsBuilder(
|
||||||
|
|||||||
@@ -135,7 +135,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
|
|||||||
self,
|
self,
|
||||||
inputs: ProcessorInputs,
|
inputs: ProcessorInputs,
|
||||||
timing_ctx: TimingContext,
|
timing_ctx: TimingContext,
|
||||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
) -> MultiModalProcessingInfo:
|
||||||
# The processor logic is different for len(images) <= 1 vs > 1
|
# The processor logic is different for len(images) <= 1 vs > 1
|
||||||
# Since the processing cache assumes that the processor output is
|
# Since the processing cache assumes that the processor output is
|
||||||
# invariant of how many images are passed per prompt, we only
|
# invariant of how many images are passed per prompt, we only
|
||||||
|
|||||||
@@ -731,28 +731,6 @@ class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo
|
|||||||
|
|
||||||
|
|
||||||
class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]):
|
class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]):
|
||||||
def _call_hf_processor(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
|
||||||
# HunYuanVLProcessor requires image placeholders wrapped with start/end tokens.
|
|
||||||
if mm_data.get("images") is not None and prompt:
|
|
||||||
img_tok = hf_processor.image_token
|
|
||||||
wrapped = (
|
|
||||||
f"{hf_processor.image_start_token}{img_tok}"
|
|
||||||
f"{hf_processor.image_end_token}"
|
|
||||||
)
|
|
||||||
if img_tok in prompt and wrapped not in prompt:
|
|
||||||
prompt = prompt.replace(img_tok, wrapped)
|
|
||||||
return self.info.ctx.call_hf_processor(
|
|
||||||
hf_processor,
|
|
||||||
dict(text=prompt, **mm_data),
|
|
||||||
mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
mm_items: MultiModalDataItems,
|
mm_items: MultiModalDataItems,
|
||||||
|
|||||||
@@ -190,77 +190,71 @@ class HCXVisionDummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionProcessingInfo
|
|||||||
|
|
||||||
|
|
||||||
class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingInfo]):
|
class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
for video_idx, video_arr in enumerate(mm_data.get("videos", [])):
|
for video_idx, video_arr in enumerate(mm_data.get("videos", [])):
|
||||||
if video_arr.dtype != np.uint8:
|
if video_arr.dtype != np.uint8:
|
||||||
mm_data["videos"][video_idx] = video_arr.astype(np.uint8)
|
mm_data["videos"][video_idx] = video_arr.astype(np.uint8)
|
||||||
|
|
||||||
processed_outputs = self.info.ctx.call_hf_processor(
|
images = mm_data.get("images")
|
||||||
hf_processor=self.info.get_hf_processor(**mm_kwargs),
|
videos = mm_data.get("videos")
|
||||||
|
|
||||||
|
# batchify input as a single item
|
||||||
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
|
hf_processor=self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
data=dict(
|
data=dict(
|
||||||
text=prompt,
|
images=None if images is None else [images],
|
||||||
images=None,
|
videos=None if videos is None else [videos],
|
||||||
videos=None,
|
|
||||||
),
|
),
|
||||||
) # text-only
|
)
|
||||||
|
|
||||||
if len(mm_data) > 0:
|
for k, v in processed_data.items():
|
||||||
images = mm_data.get("images")
|
if isinstance(v, list) and len(v) > 0:
|
||||||
videos = mm_data.get("videos")
|
assert len(v) == 1
|
||||||
|
processed_data[k] = v[0]
|
||||||
|
|
||||||
# batchify input as a single item
|
if images:
|
||||||
_processed_outputs = self.info.ctx.call_hf_processor(
|
processed_data["image_sizes_images"] = torch.tensor(
|
||||||
hf_processor=self.info.get_hf_processor(**mm_kwargs),
|
processed_data["image_sizes_images"]
|
||||||
data=dict(
|
)
|
||||||
text=None,
|
processed_data["vision_query_lengths_images"] = torch.tensor(
|
||||||
images=None if images is None else [images],
|
processed_data["vision_query_lengths_images"]
|
||||||
videos=None if videos is None else [videos],
|
)
|
||||||
),
|
|
||||||
) # mm-only
|
|
||||||
|
|
||||||
for k, v in _processed_outputs.items():
|
if videos:
|
||||||
if isinstance(v, list) and len(v) > 0:
|
_idx_per_video = [
|
||||||
assert len(v) == 1
|
0,
|
||||||
_processed_outputs[k] = v[0]
|
*accumulate(get_num_combined_frames(len(video)) for video in videos),
|
||||||
|
]
|
||||||
if images:
|
processed_data["pixel_values_videos"] = [
|
||||||
_processed_outputs["image_sizes_images"] = torch.tensor(
|
processed_data["pixel_values_videos"][
|
||||||
_processed_outputs["image_sizes_images"]
|
_idx_per_video[i] : _idx_per_video[i + 1]
|
||||||
)
|
|
||||||
_processed_outputs["vision_query_lengths_images"] = torch.tensor(
|
|
||||||
_processed_outputs["vision_query_lengths_images"]
|
|
||||||
)
|
|
||||||
|
|
||||||
if videos:
|
|
||||||
_idx_per_video = [
|
|
||||||
0,
|
|
||||||
*accumulate(
|
|
||||||
get_num_combined_frames(len(video)) for video in videos
|
|
||||||
),
|
|
||||||
]
|
]
|
||||||
_processed_outputs["pixel_values_videos"] = [
|
for i in range(len(videos))
|
||||||
_processed_outputs["pixel_values_videos"][
|
]
|
||||||
|
processed_data["vision_query_lengths_videos"] = [
|
||||||
|
torch.tensor(
|
||||||
|
processed_data["vision_query_lengths_videos"][
|
||||||
_idx_per_video[i] : _idx_per_video[i + 1]
|
_idx_per_video[i] : _idx_per_video[i + 1]
|
||||||
]
|
]
|
||||||
for i in range(len(videos))
|
)
|
||||||
]
|
for i in range(len(videos))
|
||||||
_processed_outputs["vision_query_lengths_videos"] = [
|
]
|
||||||
torch.tensor(
|
|
||||||
_processed_outputs["vision_query_lengths_videos"][
|
|
||||||
_idx_per_video[i] : _idx_per_video[i + 1]
|
|
||||||
]
|
|
||||||
)
|
|
||||||
for i in range(len(videos))
|
|
||||||
]
|
|
||||||
|
|
||||||
processed_outputs.update(_processed_outputs)
|
processed_data.update(passthrough_data)
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -261,34 +261,44 @@ class HCXVisionV2MultiModalProcessor(
|
|||||||
):
|
):
|
||||||
"""Multimodal processor for HyperCLOVAX V2 (32B Think model)."""
|
"""Multimodal processor for HyperCLOVAX V2 (32B Think model)."""
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
images = mm_data.get("images")
|
images = mm_data.get("images")
|
||||||
videos = mm_data.get("videos")
|
videos = mm_data.get("videos")
|
||||||
|
|
||||||
# Get the HF processor
|
# Get the HF processor
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
|
||||||
# Build data dict for HF processor (images/videos only)
|
# Build data dict for HF processor (images/videos only)
|
||||||
# NOTE: We pass the prompt as-is without token normalization.
|
# NOTE: We pass the prompt as-is without token normalization.
|
||||||
# Token expansion is handled by vLLM via _get_prompt_updates.
|
# Token expansion is handled by vLLM via _get_prompt_updates.
|
||||||
data: dict[str, object] = dict(
|
data: dict[str, object] = dict(
|
||||||
text=prompt,
|
text=prompt_text,
|
||||||
images=images,
|
images=images,
|
||||||
videos=videos,
|
videos=videos,
|
||||||
)
|
)
|
||||||
|
|
||||||
processed_outputs = self.info.ctx.call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
hf_processor=hf_processor,
|
hf_processor=hf_processor,
|
||||||
data=data,
|
data=data,
|
||||||
kwargs=mm_kwargs,
|
kwargs=hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -239,51 +239,58 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
|
|||||||
|
|
||||||
|
|
||||||
class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]):
|
class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
# Text-only input not supported in composite processor
|
valid_mm_items = mm_items.select(
|
||||||
if not (images := mm_data.get("images", [])):
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
)
|
||||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
image_processor = self.info.get_hf_processor().image_processor
|
image_processor = self.info.get_hf_processor().image_processor
|
||||||
if getattr(image_processor, "backend", "pil") == "pil":
|
if getattr(image_processor, "backend", "pil") == "pil":
|
||||||
mm_kwargs = {"input_data_format": "channels_last", **mm_kwargs}
|
hf_processor_mm_kwargs = {
|
||||||
|
"input_data_format": "channels_last",
|
||||||
|
**hf_processor_mm_kwargs,
|
||||||
|
}
|
||||||
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
prompt,
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
mm_data,
|
mm_data,
|
||||||
mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
images = mm_data.get("images", [])
|
||||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||||
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
||||||
image_sizes = [
|
image_sizes = [
|
||||||
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
||||||
]
|
]
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
|
||||||
num_patches = [
|
num_patches = [
|
||||||
self.info.get_num_patches(
|
self.info.get_num_patches(
|
||||||
image_width=size.width,
|
image_width=size.width,
|
||||||
image_height=size.height,
|
image_height=size.height,
|
||||||
processor=hf_processor,
|
processor=hf_processor,
|
||||||
mm_kwargs=mm_kwargs,
|
mm_kwargs=hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
for size in image_sizes
|
for size in image_sizes
|
||||||
]
|
]
|
||||||
processed_outputs["num_patches"] = torch.tensor(num_patches)
|
processed_data["num_patches"] = torch.tensor(num_patches)
|
||||||
|
|
||||||
# Remove the extra batch dimension
|
# Remove the extra batch dimension
|
||||||
processed_outputs["pixel_values"].squeeze_(0)
|
processed_data["pixel_values"].squeeze_(0)
|
||||||
processed_outputs["pixel_attention_mask"].squeeze_(0)
|
processed_data["pixel_attention_mask"].squeeze_(0)
|
||||||
|
|
||||||
return processed_outputs
|
processed_data.update(passthrough_data)
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -332,19 +332,28 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
|
|||||||
class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]):
|
class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]):
|
||||||
"""Basic image-only MultiModalProcessor for InternS1-style models."""
|
"""Basic image-only MultiModalProcessor for InternS1-style models."""
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
mm_data = dict(mm_data)
|
mm_data = dict(mm_data)
|
||||||
videos = mm_data.pop("videos", [])
|
videos = mm_data.pop("videos", [])
|
||||||
images = mm_data.pop("images", [])
|
images = mm_data.pop("images", [])
|
||||||
assert isinstance(videos, list)
|
assert isinstance(videos, list)
|
||||||
assert isinstance(images, list)
|
assert isinstance(images, list)
|
||||||
|
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
tokenizer = hf_processor.tokenizer
|
tokenizer = hf_processor.tokenizer
|
||||||
video_token_id = tokenizer.encode(
|
video_token_id = tokenizer.encode(
|
||||||
hf_processor.video_token, add_special_tokens=False
|
hf_processor.video_token, add_special_tokens=False
|
||||||
@@ -352,23 +361,29 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
|||||||
assert len(video_token_id) == 1
|
assert len(video_token_id) == 1
|
||||||
video_token_id = video_token_id[0]
|
video_token_id = video_token_id[0]
|
||||||
|
|
||||||
prompt = re.sub(hf_processor.image_token, "<image_placeholder>", prompt)
|
prompt_text = re.sub(
|
||||||
prompt = re.sub(hf_processor.video_token, "<video_placeholder>", prompt)
|
hf_processor.image_token, "<image_placeholder>", prompt_text
|
||||||
|
)
|
||||||
|
prompt_text = re.sub(
|
||||||
|
hf_processor.video_token, "<video_placeholder>", prompt_text
|
||||||
|
)
|
||||||
|
|
||||||
image_outputs = {}
|
image_outputs = {}
|
||||||
if images:
|
if images:
|
||||||
image_pixel_values = []
|
image_pixel_values = []
|
||||||
for image in images:
|
for image in images:
|
||||||
processed_outputs = super()._call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
prompt=hf_processor.image_token,
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
mm_data={"images": image},
|
dict(text=hf_processor.image_token, **{"images": image}),
|
||||||
mm_kwargs=mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
image_pixel_values.append(processed_outputs.pop("pixel_values"))
|
image_pixel_values.append(processed_data.pop("pixel_values"))
|
||||||
|
|
||||||
input_ids = processed_outputs.pop("input_ids")
|
input_ids = processed_data.pop("input_ids")
|
||||||
image_placeholder = tokenizer.batch_decode(input_ids)[0]
|
image_placeholder = tokenizer.batch_decode(input_ids)[0]
|
||||||
prompt = prompt.replace("<image_placeholder>", image_placeholder, 1)
|
prompt_text = prompt_text.replace(
|
||||||
|
"<image_placeholder>", image_placeholder, 1
|
||||||
|
)
|
||||||
|
|
||||||
num_patches = [len(item) for item in image_pixel_values]
|
num_patches = [len(item) for item in image_pixel_values]
|
||||||
image_outputs = {
|
image_outputs = {
|
||||||
@@ -381,18 +396,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
|||||||
if videos:
|
if videos:
|
||||||
video_pixel_values = []
|
video_pixel_values = []
|
||||||
for video in videos:
|
for video in videos:
|
||||||
processed_outputs = super()._call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
prompt=hf_processor.video_token,
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
mm_data={"videos": video},
|
dict(text=hf_processor.video_token, **{"videos": video}),
|
||||||
mm_kwargs=mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
video_pixel_values.append(processed_outputs.pop("pixel_values"))
|
video_pixel_values.append(processed_data.pop("pixel_values"))
|
||||||
|
|
||||||
input_ids = processed_outputs.pop("input_ids")
|
input_ids = processed_data.pop("input_ids")
|
||||||
input_ids[input_ids == hf_processor.image_token_id] = video_token_id
|
input_ids[input_ids == hf_processor.image_token_id] = video_token_id
|
||||||
|
|
||||||
video_placeholder = tokenizer.batch_decode(input_ids)[0]
|
video_placeholder = tokenizer.batch_decode(input_ids)[0]
|
||||||
prompt = prompt.replace("<video_placeholder>", video_placeholder, 1)
|
prompt_text = prompt_text.replace(
|
||||||
|
"<video_placeholder>", video_placeholder, 1
|
||||||
|
)
|
||||||
|
|
||||||
num_frames = [len(item) for item in video_pixel_values]
|
num_frames = [len(item) for item in video_pixel_values]
|
||||||
video_outputs = {
|
video_outputs = {
|
||||||
@@ -401,11 +418,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
|||||||
"video_token_id": torch.tensor(video_token_id),
|
"video_token_id": torch.tensor(video_token_id),
|
||||||
}
|
}
|
||||||
|
|
||||||
prompt = re.sub("<image_placeholder>", hf_processor.image_token, prompt)
|
prompt_text = re.sub(
|
||||||
prompt = re.sub("<video_placeholder>", hf_processor.video_token, prompt)
|
"<image_placeholder>", hf_processor.image_token, prompt_text
|
||||||
text_outputs = tokenizer(prompt, return_tensors="pt")
|
)
|
||||||
|
prompt_text = re.sub(
|
||||||
|
"<video_placeholder>", hf_processor.video_token, prompt_text
|
||||||
|
)
|
||||||
|
text_outputs = tokenizer(prompt_text, return_tensors="pt")
|
||||||
|
|
||||||
return BatchFeature({**text_outputs, **image_outputs, **video_outputs})
|
processed_data.update(text_outputs)
|
||||||
|
processed_data.update(image_outputs)
|
||||||
|
processed_data.update(video_outputs)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -220,27 +220,21 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
|
|||||||
class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||||
"""Basic image-only MultiModalProcessor for InternVL-style models."""
|
"""Basic image-only MultiModalProcessor for InternVL-style models."""
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _postprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
processed_outputs = super()._call_hf_processor(
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
|
||||||
image_token_id = hf_processor.ctx_image_token_id
|
image_token_id = hf_processor.ctx_image_token_id
|
||||||
|
|
||||||
# Since there may be extra tokens in the feature placeholders,
|
# Since there may be extra tokens in the feature placeholders,
|
||||||
# we need to pass the image token ID to the model to select the
|
# we need to pass the image token ID to the model to select the
|
||||||
# tokens to merge from the vision encoder outputs
|
# tokens to merge from the vision encoder outputs
|
||||||
processed_outputs["image_token_id"] = torch.tensor(image_token_id)
|
processed_data["image_token_id"] = torch.tensor(image_token_id)
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_image_fields_config(self, hf_inputs: BatchFeature):
|
def _get_image_fields_config(self, hf_inputs: BatchFeature):
|
||||||
image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
|
image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
|
||||||
@@ -455,19 +449,23 @@ class InternVLMultiModalProcessor(
|
|||||||
):
|
):
|
||||||
"""InternVL MultiModalProcessor extended for video support"""
|
"""InternVL MultiModalProcessor extended for video support"""
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _postprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
processed_data = super()._postprocess_hf_mm_data(
|
||||||
|
mm_data,
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
|
processed_data,
|
||||||
|
)
|
||||||
|
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
if (video_token_id := hf_processor.ctx_video_token_id) is not None:
|
if (video_token_id := hf_processor.ctx_video_token_id) is not None:
|
||||||
processed_outputs["video_token_id"] = torch.tensor(video_token_id)
|
processed_data["video_token_id"] = torch.tensor(video_token_id)
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_video_fields_config(self, hf_inputs: BatchFeature):
|
def _get_video_fields_config(self, hf_inputs: BatchFeature):
|
||||||
video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
|
video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
|
|||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||||
from vllm.model_executor.layers.pooler import DispatchPooler
|
from vllm.model_executor.layers.pooler import DispatchPooler
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
|
from vllm.multimodal.parse import MultiModalDataItems
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
|
|
||||||
from .interfaces import SupportsCrossEncoding, SupportsMultiModal, SupportsScoreTemplate
|
from .interfaces import SupportsCrossEncoding, SupportsMultiModal, SupportsScoreTemplate
|
||||||
@@ -54,19 +55,28 @@ class JinaVLScorer(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
# NOTE: We should reverse the order of the mm_data because the
|
valid_mm_items = mm_items.select(
|
||||||
# query prompt is placed after the document prompt in the score
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
# template for JinaVLForRanking model, but in mm_data they are
|
)
|
||||||
# stored in the opposite order (query first, then document).
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
for _, value in mm_data.items():
|
for _, value in mm_data.items():
|
||||||
value.reverse()
|
value.reverse()
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
|
mm_data,
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
|
|
||||||
@MULTIMODAL_REGISTRY.register_processor(
|
@MULTIMODAL_REGISTRY.register_processor(
|
||||||
|
|||||||
@@ -461,17 +461,21 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
|||||||
def media_token_id(self) -> int:
|
def media_token_id(self) -> int:
|
||||||
return self.info.get_hf_config().text_config.eos_token_id + 1
|
return self.info.get_hf_config().text_config.eos_token_id + 1
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
"""Run the underlying HF processor on text and image data."""
|
"""Run the underlying HF processor on text and image data."""
|
||||||
# Text-only input is handled as a special case here.
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
if not mm_data or not mm_data.get("images", []):
|
if not mm_data or not mm_data.get("images", []):
|
||||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
return BatchFeature(dict(passthrough_data))
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
# Images
|
# Images
|
||||||
image_inputs = mm_data.get("images", [])
|
image_inputs = mm_data.get("images", [])
|
||||||
@@ -493,7 +497,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
|||||||
|
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
media_token = tokenizer.convert_ids_to_tokens([self.media_token_id])[0]
|
media_token = tokenizer.convert_ids_to_tokens([self.media_token_id])[0]
|
||||||
prompt_replaced = prompt.replace("<image>", media_token)
|
prompt_replaced = prompt_text.replace("<image>", media_token)
|
||||||
input_ids = tokenizer.encode(prompt_replaced)
|
input_ids = tokenizer.encode(prompt_replaced)
|
||||||
input_ids = torch.tensor(input_ids)
|
input_ids = torch.tensor(input_ids)
|
||||||
|
|
||||||
@@ -527,7 +531,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
|||||||
image_token_thw=torch.tensor(image_meta["image_token_thw"]),
|
image_token_thw=torch.tensor(image_meta["image_token_thw"]),
|
||||||
pixel_sizes=torch.tensor(pixel_sizes),
|
pixel_sizes=torch.tensor(pixel_sizes),
|
||||||
)
|
)
|
||||||
return BatchFeature(combined_outputs, tensor_type="pt")
|
processed_data = BatchFeature(combined_outputs, tensor_type="pt")
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -1162,15 +1162,8 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
|
class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
# Override to use the text path instead of token path to use the
|
|
||||||
# video-specific logic in processing_keye.py
|
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -378,15 +378,8 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
|
|||||||
|
|
||||||
|
|
||||||
class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
|
class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
# Override to use the text path instead of token path to use the
|
|
||||||
# video-specific logic in processing_keye.py
|
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ from vllm.multimodal.parse import (
|
|||||||
DictEmbeddingItems,
|
DictEmbeddingItems,
|
||||||
ModalityData,
|
ModalityData,
|
||||||
ModalityDataItems,
|
ModalityDataItems,
|
||||||
|
MultiModalDataItems,
|
||||||
MultiModalDataParser,
|
MultiModalDataParser,
|
||||||
)
|
)
|
||||||
from vllm.multimodal.processing import (
|
from vllm.multimodal.processing import (
|
||||||
@@ -226,13 +227,20 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser):
|
|||||||
class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]):
|
class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]):
|
||||||
"""vLLM multi-modal processor wrapper for Kimi-Audio."""
|
"""vLLM multi-modal processor wrapper for Kimi-Audio."""
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
"""Call the HuggingFace processor."""
|
"""Call the HuggingFace processor."""
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
# Convert mm_data format: {'audios': [...]} -> {'audio': ...}
|
# Convert mm_data format: {'audios': [...]} -> {'audio': ...}
|
||||||
mm_data = dict(mm_data)
|
mm_data = dict(mm_data)
|
||||||
audios = mm_data.pop("audios", [])
|
audios = mm_data.pop("audios", [])
|
||||||
@@ -252,11 +260,13 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
|
|||||||
mm_data["audio"] = audio_arrays
|
mm_data["audio"] = audio_arrays
|
||||||
|
|
||||||
# Use the context's call_hf_processor for proper handling
|
# Use the context's call_hf_processor for proper handling
|
||||||
return self.info.ctx.call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
self.info.get_hf_processor(**mm_kwargs),
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
dict(text=prompt, **mm_data),
|
mm_data,
|
||||||
mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -258,16 +258,6 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
|
|||||||
grid_thws=MultiModalFieldConfig.batched("vision_chunk", keep_on_cpu=True),
|
grid_thws=MultiModalFieldConfig.batched("vision_chunk", keep_on_cpu=True),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _call_hf_processor(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
# Override to use the text path instead of token path because vision chunk
|
|
||||||
# is not considered
|
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
mm_items: MultiModalDataItems,
|
mm_items: MultiModalDataItems,
|
||||||
|
|||||||
@@ -237,6 +237,9 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
|
class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
|
||||||
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
hf_inputs: BatchFeature,
|
hf_inputs: BatchFeature,
|
||||||
|
|||||||
@@ -393,45 +393,38 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
|
class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
# Text-only input not supported in composite processor
|
if not mm_data:
|
||||||
if not (images := mm_data.get("images", [])):
|
return processed_data
|
||||||
prompt_ids = self.info.get_tokenizer().encode(
|
|
||||||
prompt, add_special_tokens=False
|
|
||||||
)
|
|
||||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
|
||||||
prompt,
|
|
||||||
mm_data,
|
|
||||||
mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
images = mm_data.get("images", [])
|
||||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||||
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
||||||
image_sizes = [
|
image_sizes = [
|
||||||
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
||||||
]
|
]
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
|
||||||
num_patches = [
|
num_patches = [
|
||||||
self.info.get_num_patches(
|
self.info.get_num_patches(
|
||||||
image_width=size.width,
|
image_width=size.width,
|
||||||
image_height=size.height,
|
image_height=size.height,
|
||||||
processor=hf_processor,
|
processor=hf_processor,
|
||||||
mm_kwargs=mm_kwargs,
|
mm_kwargs=hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
for size in image_sizes
|
for size in image_sizes
|
||||||
]
|
]
|
||||||
processed_outputs["num_patches"] = torch.tensor(num_patches)
|
processed_data["num_patches"] = torch.tensor(num_patches)
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -39,20 +39,19 @@ _I = TypeVar("_I", bound=Mistral3ProcessingInfo)
|
|||||||
|
|
||||||
|
|
||||||
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
# NOTE: LightOnOCR does not use break/end tokens, so we remove them here.
|
def _postprocess_hf_mm_data(
|
||||||
input_ids = processed_outputs.get("input_ids")
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if not mm_data:
|
||||||
|
return processed_data
|
||||||
|
|
||||||
|
input_ids = processed_data.get("input_ids")
|
||||||
if input_ids is not None:
|
if input_ids is not None:
|
||||||
processor = self.info.get_hf_processor()
|
processor = self.info.get_hf_processor()
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
@@ -67,22 +66,22 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
|
|||||||
torch.tensor([break_id, end_id]),
|
torch.tensor([break_id, end_id]),
|
||||||
)
|
)
|
||||||
|
|
||||||
processed_outputs["input_ids"] = input_ids[keep_mask].unsqueeze(0)
|
processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0)
|
||||||
if "attention_mask" in processed_outputs:
|
if "attention_mask" in processed_data:
|
||||||
processed_outputs["attention_mask"] = processed_outputs[
|
processed_data["attention_mask"] = processed_data["attention_mask"][
|
||||||
"attention_mask"
|
keep_mask
|
||||||
][keep_mask].unsqueeze(0)
|
].unsqueeze(0)
|
||||||
|
|
||||||
# un-pad pixel_values per-image so caches remain independent.
|
# un-pad pixel_values per-image so caches remain independent.
|
||||||
pixel_values = processed_outputs.get("pixel_values")
|
pixel_values = processed_data.get("pixel_values")
|
||||||
if pixel_values is not None:
|
if pixel_values is not None:
|
||||||
image_sizes = processed_outputs["image_sizes"]
|
image_sizes = processed_data["image_sizes"]
|
||||||
assert len(pixel_values) == len(image_sizes)
|
assert len(pixel_values) == len(image_sizes)
|
||||||
processed_outputs["pixel_values"] = [
|
processed_data["pixel_values"] = [
|
||||||
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
|
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
|
||||||
]
|
]
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -321,30 +321,30 @@ class PixtralHFProcessingInfo(BaseLlavaProcessingInfo):
|
|||||||
|
|
||||||
|
|
||||||
class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]):
|
class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
pixel_values = processed_outputs.get("pixel_values")
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if not mm_data:
|
||||||
|
return processed_data
|
||||||
|
|
||||||
|
pixel_values = processed_data.get("pixel_values")
|
||||||
if pixel_values is not None:
|
if pixel_values is not None:
|
||||||
# Avoid padding since we need the output for each image to be
|
# Avoid padding since we need the output for each image to be
|
||||||
# independent of other images for the cache to work correctly
|
# independent of other images for the cache to work correctly
|
||||||
image_sizes = processed_outputs["image_sizes"]
|
image_sizes = processed_data["image_sizes"]
|
||||||
assert len(pixel_values) == len(image_sizes)
|
assert len(pixel_values) == len(image_sizes)
|
||||||
|
|
||||||
processed_outputs["pixel_values"] = [
|
processed_data["pixel_values"] = [
|
||||||
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
|
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
|
||||||
]
|
]
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -211,6 +211,9 @@ class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]):
|
|||||||
|
|
||||||
|
|
||||||
class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):
|
class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):
|
||||||
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
hf_inputs: BatchFeature,
|
hf_inputs: BatchFeature,
|
||||||
|
|||||||
@@ -321,22 +321,33 @@ class LlavaOnevisionMultiModalProcessor(
|
|||||||
pixel_values_videos=MultiModalFieldConfig.batched("video"),
|
pixel_values_videos=MultiModalFieldConfig.batched("video"),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
mm_data = dict(mm_data)
|
mm_data = dict(mm_data)
|
||||||
videos = mm_data.pop("videos", [])
|
videos = mm_data.pop("videos", [])
|
||||||
assert isinstance(videos, list)
|
assert isinstance(videos, list)
|
||||||
|
|
||||||
if not videos:
|
if not videos:
|
||||||
return super()._call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
prompt=prompt,
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
mm_data=mm_data,
|
dict(text=prompt_text, **mm_data),
|
||||||
mm_kwargs=mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
# LLaVA-OneVision processor doesn't support multiple videos
|
# LLaVA-OneVision processor doesn't support multiple videos
|
||||||
# with different sizes when converting back to tensors
|
# with different sizes when converting back to tensors
|
||||||
@@ -346,19 +357,13 @@ class LlavaOnevisionMultiModalProcessor(
|
|||||||
image_token = processor.image_token
|
image_token = processor.image_token
|
||||||
video_token = processor.video_token
|
video_token = processor.video_token
|
||||||
|
|
||||||
text_outputs = super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data={},
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
images = mm_data.pop("images", [])
|
images = mm_data.pop("images", [])
|
||||||
assert isinstance(images, list)
|
assert isinstance(images, list)
|
||||||
if images:
|
if images:
|
||||||
processor_outputs = super()._call_hf_processor(
|
processor_outputs = self.info.ctx.call_hf_processor(
|
||||||
prompt=image_token * len(images),
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
mm_data={"images": images},
|
dict(text=image_token * len(images), **{"images": images}),
|
||||||
mm_kwargs=mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
image_outputs = {
|
image_outputs = {
|
||||||
k: v
|
k: v
|
||||||
@@ -370,10 +375,10 @@ class LlavaOnevisionMultiModalProcessor(
|
|||||||
|
|
||||||
pixel_values_videos = []
|
pixel_values_videos = []
|
||||||
for video in videos:
|
for video in videos:
|
||||||
item_outputs = super()._call_hf_processor(
|
item_outputs = self.info.ctx.call_hf_processor(
|
||||||
prompt=video_token,
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
mm_data={"videos": video},
|
dict(text=video_token, **{"videos": video}),
|
||||||
mm_kwargs=mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
|
pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
|
||||||
@@ -381,11 +386,13 @@ class LlavaOnevisionMultiModalProcessor(
|
|||||||
video_outputs = {"pixel_values_videos": pixel_values_videos}
|
video_outputs = {"pixel_values_videos": pixel_values_videos}
|
||||||
|
|
||||||
combined_outputs = dict(
|
combined_outputs = dict(
|
||||||
text_outputs,
|
input_ids=prompt_text,
|
||||||
**image_outputs,
|
**image_outputs,
|
||||||
**video_outputs,
|
**video_outputs,
|
||||||
)
|
)
|
||||||
return BatchFeature(combined_outputs)
|
processed_data = BatchFeature(combined_outputs)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -286,7 +286,7 @@ def _validate_video_sources(paths, model_config) -> list[str]:
|
|||||||
# participates in the standard decode-stage pipeline.
|
# participates in the standard decode-stage pipeline.
|
||||||
#
|
#
|
||||||
# * codec backend: NOT a loader. OV2's codec path needs the video path string
|
# * codec backend: NOT a loader. OV2's codec path needs the video path string
|
||||||
# to survive into ``_call_hf_processor``, where the HF processor builds the
|
# to survive into ``_apply_hf_processor_main``, where the HF processor builds the
|
||||||
# codec canvas + smart_resize + patchify
|
# codec canvas + smart_resize + patchify
|
||||||
# (pixel_values/image_grid_thw/patch_positions). That transform is
|
# (pixel_values/image_grid_thw/patch_positions). That transform is
|
||||||
# path-level and inseparable; it cannot be reconstructed from pre-decoded
|
# path-level and inseparable; it cannot be reconstructed from pre-decoded
|
||||||
@@ -302,7 +302,7 @@ def prepare_codec_video_input(video_path: str) -> tuple:
|
|||||||
|
|
||||||
Returns ``(dummy_ndarray, metadata)`` where the ndarray satisfies the
|
Returns ``(dummy_ndarray, metadata)`` where the ndarray satisfies the
|
||||||
parser's 4-D shape check and the metadata carries the actual path to
|
parser's 4-D shape check and the metadata carries the actual path to
|
||||||
our ``_call_hf_processor``. Use as::
|
our ``_apply_hf_processor_main``. Use as::
|
||||||
|
|
||||||
multi_modal_data = {"video": prepare_codec_video_input("foo.mp4")}
|
multi_modal_data = {"video": prepare_codec_video_input("foo.mp4")}
|
||||||
|
|
||||||
@@ -1283,7 +1283,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
|
|||||||
# ``video_needs_metadata=True`` makes the parser preserve both the
|
# ``video_needs_metadata=True`` makes the parser preserve both the
|
||||||
# ``(frames, metadata)`` tuples from the frame backend and the
|
# ``(frames, metadata)`` tuples from the frame backend and the
|
||||||
# ``(dummy, {marker: path})`` tuples from prepare_codec_video_input;
|
# ``(dummy, {marker: path})`` tuples from prepare_codec_video_input;
|
||||||
# both are dispatched by metadata content in ``_call_hf_processor``.
|
# both are dispatched by metadata content in ``_apply_hf_processor_main``.
|
||||||
return LlavaOnevision2MultiModalDataParser(
|
return LlavaOnevision2MultiModalDataParser(
|
||||||
self.get_hf_config().vision_config.spatial_merge_size,
|
self.get_hf_config().vision_config.spatial_merge_size,
|
||||||
video_needs_metadata=True,
|
video_needs_metadata=True,
|
||||||
@@ -1547,18 +1547,23 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
|
|||||||
class LlavaOnevision2MultiModalProcessor(
|
class LlavaOnevision2MultiModalProcessor(
|
||||||
BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]
|
BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
# The wrapped OV2 processor is a bare custom class without the standard
|
valid_mm_items = mm_items.select(
|
||||||
# ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
# fails; overriding this method routes the base class to call us
|
)
|
||||||
# directly.
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
|
||||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||||
merged_kwargs.setdefault("return_tensors", "pt")
|
merged_kwargs.setdefault("return_tensors", "pt")
|
||||||
call_kwargs = {
|
call_kwargs = {
|
||||||
k: v
|
k: v
|
||||||
@@ -1627,21 +1632,25 @@ class LlavaOnevision2MultiModalProcessor(
|
|||||||
if len(codec_video_paths) > 1
|
if len(codec_video_paths) > 1
|
||||||
else codec_video_paths[0]
|
else codec_video_paths[0]
|
||||||
)
|
)
|
||||||
# Route through the base ``_call_hf_processor`` so float-tensor
|
# Route through ``ctx.call_hf_processor`` so float-tensor
|
||||||
# dtype postprocessing is applied automatically; inject
|
# dtype postprocessing is applied automatically; inject
|
||||||
# ``video_backend="codec"`` via mm_kwargs so the wrapped processor
|
# ``video_backend="codec"`` via mm_kwargs so the wrapped processor
|
||||||
# dispatches to its codec branch.
|
# dispatches to its codec branch.
|
||||||
output = super()._call_hf_processor(
|
output = self.info.ctx.call_hf_processor(
|
||||||
prompt=prompt,
|
self.info.get_hf_processor(
|
||||||
mm_data=mm_data,
|
**{**hf_processor_mm_kwargs, "video_backend": "codec"}
|
||||||
mm_kwargs={**mm_kwargs, "video_backend": "codec"},
|
),
|
||||||
|
dict(text=prompt_text, **mm_data),
|
||||||
|
{**hf_processor_mm_kwargs, "video_backend": "codec"},
|
||||||
)
|
)
|
||||||
data = dict(output)
|
data = dict(output)
|
||||||
return BatchFeature(
|
processed_data = BatchFeature(
|
||||||
self._rename_codec_outputs_to_video(
|
self._rename_codec_outputs_to_video(
|
||||||
data, codec_video_paths, hf_processor
|
data, codec_video_paths, hf_processor
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
# ---- Frame backend (registered LlavaOnevision2VideoBackend) ------
|
# ---- Frame backend (registered LlavaOnevision2VideoBackend) ------
|
||||||
# Every non-codec video reaches here as a ``(frames_ndarray, metadata)``
|
# Every non-codec video reaches here as a ``(frames_ndarray, metadata)``
|
||||||
@@ -1657,7 +1666,9 @@ class LlavaOnevision2MultiModalProcessor(
|
|||||||
# local-file gating is enforced by the connector before decoding.
|
# local-file gating is enforced by the connector before decoding.
|
||||||
if videos_present:
|
if videos_present:
|
||||||
timestamp_decimals = int(
|
timestamp_decimals = int(
|
||||||
mm_kwargs.get("timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS)
|
hf_processor_mm_kwargs.get(
|
||||||
|
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
per_video_frames: list[list[Image.Image]] = []
|
per_video_frames: list[list[Image.Image]] = []
|
||||||
@@ -1671,7 +1682,7 @@ class LlavaOnevision2MultiModalProcessor(
|
|||||||
# ``<{t} seconds><|vision_start|><|image_pad|><|vision_end|>``
|
# ``<{t} seconds><|vision_start|><|image_pad|><|vision_end|>``
|
||||||
# blocks (matches the OV2 hf-chat reference exactly).
|
# blocks (matches the OV2 hf-chat reference exactly).
|
||||||
new_prompt = _expand_video_markers_in_prompt(
|
new_prompt = _expand_video_markers_in_prompt(
|
||||||
prompt,
|
prompt_text,
|
||||||
per_video_timestamps,
|
per_video_timestamps,
|
||||||
timestamp_decimals=timestamp_decimals,
|
timestamp_decimals=timestamp_decimals,
|
||||||
)
|
)
|
||||||
@@ -1700,7 +1711,7 @@ class LlavaOnevision2MultiModalProcessor(
|
|||||||
row_is_video: list[bool] = []
|
row_is_video: list[bool] = []
|
||||||
vid_idx = 0
|
vid_idx = 0
|
||||||
img_idx = 0
|
img_idx = 0
|
||||||
for marker in marker_pattern.finditer(prompt):
|
for marker in marker_pattern.finditer(prompt_text):
|
||||||
if marker.lastgroup == "video":
|
if marker.lastgroup == "video":
|
||||||
frames = per_video_frames[vid_idx]
|
frames = per_video_frames[vid_idx]
|
||||||
vid_idx += 1
|
vid_idx += 1
|
||||||
@@ -1723,15 +1734,15 @@ class LlavaOnevision2MultiModalProcessor(
|
|||||||
merged_mm_data.pop("videos", None)
|
merged_mm_data.pop("videos", None)
|
||||||
merged_mm_data["images"] = flat_frames
|
merged_mm_data["images"] = flat_frames
|
||||||
|
|
||||||
# Route through the base ``_call_hf_processor`` (applies float-tensor
|
# Route through ``ctx.call_hf_processor`` (applies float-tensor
|
||||||
# dtype postprocessing automatically). The wrapped processor's image
|
# dtype postprocessing automatically). The wrapped processor's image
|
||||||
# branch ignores video/codec-only kwargs and does not forward extra
|
# branch ignores video/codec-only kwargs and does not forward extra
|
||||||
# **kwargs to the image processor, so passing the full merged kwarg
|
# **kwargs to the image processor, so passing the full merged kwarg
|
||||||
# set here is a no-op beyond return_tensors/padding.
|
# set here is a no-op beyond return_tensors/padding.
|
||||||
output = super()._call_hf_processor(
|
output = self.info.ctx.call_hf_processor(
|
||||||
prompt=new_prompt,
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
mm_data=merged_mm_data,
|
dict(text=new_prompt, **merged_mm_data),
|
||||||
mm_kwargs=mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
data = dict(output)
|
data = dict(output)
|
||||||
|
|
||||||
@@ -1788,17 +1799,21 @@ class LlavaOnevision2MultiModalProcessor(
|
|||||||
(len(per_video_timestamps),), dtype=torch.long
|
(len(per_video_timestamps),), dtype=torch.long
|
||||||
)
|
)
|
||||||
|
|
||||||
return BatchFeature(data)
|
processed_data = BatchFeature(data)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
# ---- Image-only / text-only call --------------------------------
|
# ---- Image-only / text-only call --------------------------------
|
||||||
# No videos present: delegate to the base ``_call_hf_processor``, which
|
# No videos present: delegate to ``ctx.call_hf_processor``, which
|
||||||
# runs the wrapped processor over the (possibly empty) image set and
|
# runs the wrapped processor over the (possibly empty) image set and
|
||||||
# applies float-tensor dtype postprocessing automatically.
|
# applies float-tensor dtype postprocessing automatically.
|
||||||
return super()._call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
prompt=prompt,
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
mm_data=mm_data,
|
dict(text=prompt_text, **mm_data),
|
||||||
mm_kwargs=mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _rename_codec_outputs_to_video(
|
def _rename_codec_outputs_to_video(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -26,7 +26,7 @@
|
|||||||
import collections
|
import collections
|
||||||
import collections.abc
|
import collections.abc
|
||||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||||
from typing import Annotated, Any, TypeAlias, cast
|
from typing import Annotated, TypeAlias, cast
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
@@ -583,15 +583,17 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
|
|||||||
class MiDashengLMMultiModalProcessor(
|
class MiDashengLMMultiModalProcessor(
|
||||||
BaseMultiModalProcessor[MiDashengLMProcessingInfo]
|
BaseMultiModalProcessor[MiDashengLMProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, Any],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
) -> BatchFeature:
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
|
mm_data = dict(mm_data)
|
||||||
audios = mm_data.pop("audios", [])
|
audios = mm_data.pop("audios", [])
|
||||||
|
|
||||||
# + Padding
|
|
||||||
min_audio_len = self.info.get_min_audio_len()
|
min_audio_len = self.info.get_min_audio_len()
|
||||||
processed_audios = [
|
processed_audios = [
|
||||||
np.pad(
|
np.pad(
|
||||||
@@ -608,20 +610,7 @@ class MiDashengLMMultiModalProcessor(
|
|||||||
if processed_audios:
|
if processed_audios:
|
||||||
mm_data["audio"] = processed_audios
|
mm_data["audio"] = processed_audios
|
||||||
|
|
||||||
if not mm_data.get("audio", []):
|
return mm_data, hf_processor_mm_kwargs
|
||||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
|
||||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
|
||||||
mm_kwargs = dict(
|
|
||||||
**mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
return super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -912,16 +912,23 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
|||||||
fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")
|
fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")
|
||||||
return fields
|
return fields
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.
|
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.
|
||||||
Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses
|
Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses
|
||||||
the singular form.
|
the singular form.
|
||||||
"""
|
"""
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
# Remap audios → audio (MiMoOmniProcessor uses singular param name)
|
# Remap audios → audio (MiMoOmniProcessor uses singular param name)
|
||||||
if "audios" in mm_data:
|
if "audios" in mm_data:
|
||||||
mm_data = {**mm_data, "audio": mm_data["audios"]}
|
mm_data = {**mm_data, "audio": mm_data["audios"]}
|
||||||
@@ -999,7 +1006,13 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
|||||||
|
|
||||||
mm_data = {**mm_data, "videos": converted}
|
mm_data = {**mm_data, "videos": converted}
|
||||||
|
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
|
mm_data,
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -448,7 +448,7 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
|||||||
if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems):
|
if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems):
|
||||||
audio_inputs = {}
|
audio_inputs = {}
|
||||||
else:
|
else:
|
||||||
audio_inputs = self._base_call_hf_processor(
|
audio_inputs = self._call_hf_processor_on_prompts(
|
||||||
prompts=[self.info.audio_pattern] * len(parsed_audios),
|
prompts=[self.info.audio_pattern] * len(parsed_audios),
|
||||||
mm_data={"audios": [[audio] for audio in parsed_audios]},
|
mm_data={"audios": [[audio] for audio in parsed_audios]},
|
||||||
mm_kwargs={**mm_kwargs, "chunk_input": True},
|
mm_kwargs={**mm_kwargs, "chunk_input": True},
|
||||||
|
|||||||
@@ -881,7 +881,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
|||||||
if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
|
if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
|
||||||
image_inputs = {}
|
image_inputs = {}
|
||||||
else:
|
else:
|
||||||
image_inputs = self._base_call_hf_processor(
|
image_inputs = self._call_hf_processor_on_prompts(
|
||||||
prompts=[self.info.image_pattern] * len(parsed_images),
|
prompts=[self.info.image_pattern] * len(parsed_images),
|
||||||
mm_data={"images": [[image] for image in parsed_images]},
|
mm_data={"images": [[image] for image in parsed_images]},
|
||||||
mm_kwargs=mm_kwargs,
|
mm_kwargs=mm_kwargs,
|
||||||
@@ -906,7 +906,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
|||||||
if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
|
if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
|
||||||
video_inputs = {}
|
video_inputs = {}
|
||||||
else:
|
else:
|
||||||
video_inputs = self._base_call_hf_processor(
|
video_inputs = self._call_hf_processor_on_prompts(
|
||||||
prompts=[
|
prompts=[
|
||||||
self.info.image_pattern * len(video) for video in parsed_videos
|
self.info.image_pattern * len(video) for video in parsed_videos
|
||||||
],
|
],
|
||||||
@@ -983,7 +983,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
|||||||
|
|
||||||
return new_token_ids, placeholders
|
return new_token_ids, placeholders
|
||||||
|
|
||||||
def _base_call_hf_processor(
|
def _call_hf_processor_on_prompts(
|
||||||
self,
|
self,
|
||||||
prompts: list[str],
|
prompts: list[str],
|
||||||
mm_data: Mapping[str, Sequence[object]],
|
mm_data: Mapping[str, Sequence[object]],
|
||||||
@@ -993,19 +993,19 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
|||||||
) -> dict[str, NestedTensors]:
|
) -> dict[str, NestedTensors]:
|
||||||
# This processor supports zipping prompt and mm_data together
|
# This processor supports zipping prompt and mm_data together
|
||||||
if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}:
|
if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}:
|
||||||
inputs = super()._call_hf_processor(
|
inputs = self.info.ctx.call_hf_processor(
|
||||||
prompt=prompts, # type: ignore
|
self.info.get_hf_processor(**mm_kwargs),
|
||||||
mm_data=mm_data,
|
dict(text=prompts, **mm_data),
|
||||||
mm_kwargs=mm_kwargs,
|
mm_kwargs,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
inputs = defaultdict[str, list[torch.Tensor]](list)
|
inputs = defaultdict[str, list[torch.Tensor]](list)
|
||||||
|
|
||||||
for i, prompt in enumerate(prompts):
|
for i, prompt in enumerate(prompts):
|
||||||
inputs_one = super()._call_hf_processor(
|
inputs_one = self.info.ctx.call_hf_processor(
|
||||||
prompt=prompt,
|
self.info.get_hf_processor(**mm_kwargs),
|
||||||
mm_data={k: v[i] for k, v in mm_data.items()},
|
dict(text=prompt, **{k: v[i] for k, v in mm_data.items()}),
|
||||||
mm_kwargs=mm_kwargs,
|
mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
for k, v in inputs_one.items():
|
for k, v in inputs_one.items():
|
||||||
@@ -1014,23 +1014,31 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
|||||||
|
|
||||||
return {k: inputs[k] for k in out_keys}
|
return {k: inputs[k] for k in out_keys}
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
|
|
||||||
input_ids = torch.tensor([tokenizer.encode(prompt)])
|
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
|
||||||
mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs)
|
mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
|
||||||
|
|
||||||
return BatchFeature(
|
processed_data = BatchFeature(
|
||||||
{
|
{
|
||||||
"input_ids": input_ids,
|
"input_ids": input_ids,
|
||||||
**mm_inputs,
|
**mm_inputs,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -233,30 +233,30 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo])
|
|||||||
|
|
||||||
|
|
||||||
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
pixel_values = processed_outputs.get("pixel_values")
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if not mm_data:
|
||||||
|
return processed_data
|
||||||
|
|
||||||
|
pixel_values = processed_data.get("pixel_values")
|
||||||
if pixel_values is not None:
|
if pixel_values is not None:
|
||||||
# Avoid padding since we need the output for each image to be
|
# Avoid padding since we need the output for each image to be
|
||||||
# independent of other images for the cache to work correctly
|
# independent of other images for the cache to work correctly
|
||||||
image_sizes = processed_outputs["image_sizes"]
|
image_sizes = processed_data["image_sizes"]
|
||||||
assert len(pixel_values) == len(image_sizes)
|
assert len(pixel_values) == len(image_sizes)
|
||||||
|
|
||||||
processed_outputs["pixel_values"] = [
|
processed_data["pixel_values"] = [
|
||||||
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
|
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
|
||||||
]
|
]
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -589,23 +589,23 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
|
|||||||
|
|
||||||
|
|
||||||
class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]):
|
class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if not mm_data:
|
||||||
|
return processed_data
|
||||||
|
|
||||||
|
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
image_processor = processor.image_processor
|
image_processor = processor.image_processor
|
||||||
vision_config = self.info.get_hf_config().vision_config
|
vision_config = self.info.get_hf_config().vision_config
|
||||||
|
|
||||||
if processed_outputs.get("pixel_values") is not None:
|
if processed_data.get("pixel_values") is not None:
|
||||||
assert "images" in mm_data, (
|
assert "images" in mm_data, (
|
||||||
"images expected to be in mm_data when pixel_values is present"
|
"images expected to be in mm_data when pixel_values is present"
|
||||||
)
|
)
|
||||||
@@ -636,10 +636,10 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
|
|||||||
1 if r_h * r_w == 1 else 1 + r_h * r_w for (r_h, r_w) in aspect_ratios
|
1 if r_h * r_w == 1 else 1 + r_h * r_w for (r_h, r_w) in aspect_ratios
|
||||||
]
|
]
|
||||||
|
|
||||||
processed_outputs["aspect_ratios"] = torch.tensor(aspect_ratios)
|
processed_data["aspect_ratios"] = torch.tensor(aspect_ratios)
|
||||||
processed_outputs["patches_per_image"] = torch.tensor(patches_per_image)
|
processed_data["patches_per_image"] = torch.tensor(patches_per_image)
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -1123,17 +1123,44 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||||
|
processor = self.info.get_hf_processor()
|
||||||
|
|
||||||
|
# The chat template is already applied to the prompt tokens
|
||||||
|
# Use message_format="none" to avoid applying it again
|
||||||
|
# Prepend an empty space if `always_start_with_space` is True
|
||||||
|
tokens = processor.get_tokens_input(
|
||||||
|
self.info.get_tokenizer().decode(prompt),
|
||||||
|
message_format="none",
|
||||||
|
always_start_with_space=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Prepend a BOS token id to the tokens
|
||||||
|
return self.info.ctx.call_hf_processor(
|
||||||
|
processor.process,
|
||||||
|
dict(tokens=tokens),
|
||||||
|
)["input_ids"].tolist()
|
||||||
|
|
||||||
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
mm_counts = mm_items.get_all_counts()
|
||||||
processed_outputs = self.info.ctx.call_hf_processor(
|
|
||||||
|
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
|
||||||
|
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not processor_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
hf_processor.process,
|
hf_processor.process,
|
||||||
dict(text=prompt, **mm_data),
|
dict(text=prompt_text, **processor_data),
|
||||||
mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
tokenizer = hf_processor.tokenizer
|
tokenizer = hf_processor.tokenizer
|
||||||
@@ -1141,17 +1168,16 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
|||||||
|
|
||||||
image_processor = hf_processor.image_processor
|
image_processor = hf_processor.image_processor
|
||||||
|
|
||||||
input_ids: torch.Tensor = processed_outputs.pop("input_ids")
|
processed_data.pop("input_ids")
|
||||||
processed_outputs["input_ids"] = input_ids.unsqueeze(0)
|
|
||||||
|
|
||||||
if (images := mm_data.get("images")) is not None:
|
if (images := processor_data.get("images")) is not None:
|
||||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||||
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
||||||
image_sizes = [
|
image_sizes = [
|
||||||
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
||||||
]
|
]
|
||||||
|
|
||||||
feat_is_patch = processed_outputs["image_input_idx"] >= 0
|
feat_is_patch = processed_data["image_input_idx"] >= 0
|
||||||
|
|
||||||
tilings = [
|
tilings = [
|
||||||
self.info.select_tiling(
|
self.info.select_tiling(
|
||||||
@@ -1165,35 +1191,12 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
|||||||
num_crops = torch.tensor(tilings).prod(-1) + 1
|
num_crops = torch.tensor(tilings).prod(-1) + 1
|
||||||
assert num_crops.sum() == len(feat_is_patch)
|
assert num_crops.sum() == len(feat_is_patch)
|
||||||
|
|
||||||
processed_outputs["num_crops"] = num_crops
|
processed_data["num_crops"] = num_crops
|
||||||
processed_outputs["img_patch_id"] = image_patch_id
|
processed_data["img_patch_id"] = image_patch_id
|
||||||
|
|
||||||
return processed_outputs
|
processed_data.update(passthrough_data)
|
||||||
|
|
||||||
def _apply_hf_processor_tokens_only(
|
return processed_data
|
||||||
self,
|
|
||||||
prompt_tokens: list[int],
|
|
||||||
) -> list[int]:
|
|
||||||
processor = self.info.get_hf_processor()
|
|
||||||
|
|
||||||
# The chat template is already applied to the prompt tokens
|
|
||||||
# Use message_format="none" to avoid applying it again
|
|
||||||
# Prepend an empty space if `always_start_with_space` is True
|
|
||||||
tokens = processor.get_tokens_input(
|
|
||||||
self.info.get_tokenizer().decode(prompt_tokens),
|
|
||||||
message_format="none",
|
|
||||||
always_start_with_space=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Prepend a BOS token id to the tokens
|
|
||||||
processed_data = self.info.ctx.call_hf_processor(
|
|
||||||
processor.process,
|
|
||||||
dict(tokens=tokens),
|
|
||||||
)
|
|
||||||
prompt_ids = processed_data.pop("input_ids").tolist()
|
|
||||||
print(prompt_ids, len(prompt_ids))
|
|
||||||
|
|
||||||
return prompt_ids
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -1929,30 +1929,33 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||||
def _apply_hf_processor_tokens_only(
|
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||||
self,
|
|
||||||
prompt_tokens: list[int],
|
|
||||||
) -> list[int]:
|
|
||||||
processor = self.info.get_hf_processor()
|
processor = self.info.get_hf_processor()
|
||||||
tokenizer = processor.tokenizer
|
tokenizer = processor.tokenizer
|
||||||
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
|
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
|
||||||
|
|
||||||
if len(prompt_tokens) == 0 or prompt_tokens[0] != bos_token_id:
|
if len(prompt) == 0 or prompt[0] != bos_token_id:
|
||||||
# Prepend the bos token to the prompt tokens
|
# Prepend the bos token to the prompt tokens
|
||||||
prompt_tokens = [bos_token_id] + prompt_tokens
|
prompt = [bos_token_id] + prompt
|
||||||
|
|
||||||
return prompt_tokens
|
return prompt
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
mm_data = dict(mm_data)
|
mm_counts = mm_items.get_all_counts()
|
||||||
|
|
||||||
|
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
|
||||||
|
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
mm_data = dict(processor_data)
|
||||||
|
|
||||||
hf_config = self.info.get_hf_config()
|
hf_config = self.info.get_hf_config()
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
|
||||||
def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature:
|
def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature:
|
||||||
res = hf_processor(text=text, images=images, videos=videos, **kwargs)
|
res = hf_processor(text=text, images=images, videos=videos, **kwargs)
|
||||||
@@ -1984,14 +1987,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
|||||||
# NOTE: metadata.frames_indices indicates
|
# NOTE: metadata.frames_indices indicates
|
||||||
# the sampled frames indices of pre-sampled videos, which is
|
# the sampled frames indices of pre-sampled videos, which is
|
||||||
# used to calculate the timestamps. Make sure that
|
# used to calculate the timestamps. Make sure that
|
||||||
# do_sample_frames in mm_kwargs is false for presampled videos.
|
# do_sample_frames in hf_processor_mm_kwargs is false for
|
||||||
|
# presampled videos.
|
||||||
|
|
||||||
# NOTE: a copy of mm_kwargs is created to update do_sample_frames,
|
# NOTE: a copy of hf_processor_mm_kwargs is created to update
|
||||||
# otherwise mm_hash for the object will be incorrect.
|
# do_sample_frames, otherwise mm_hash for the object will be
|
||||||
video_mm_kwargs = dict(**mm_kwargs)
|
# incorrect.
|
||||||
|
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
|
||||||
if "do_sample_frames" not in video_mm_kwargs:
|
if "do_sample_frames" not in video_mm_kwargs:
|
||||||
# molmo_utils already has "do_sample_frames" in
|
# molmo_utils already has "do_sample_frames" in
|
||||||
# mm_kwargs, don't overwrite it.
|
# hf_processor_mm_kwargs, don't overwrite it.
|
||||||
video_mm_kwargs["do_sample_frames"] = metadata.get(
|
video_mm_kwargs["do_sample_frames"] = metadata.get(
|
||||||
"do_sample_frames", False
|
"do_sample_frames", False
|
||||||
)
|
)
|
||||||
@@ -2015,7 +2020,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
|||||||
input_ids = input_ids[:, 1:]
|
input_ids = input_ids[:, 1:]
|
||||||
|
|
||||||
video_string = tokenizer.batch_decode(input_ids)[0]
|
video_string = tokenizer.batch_decode(input_ids)[0]
|
||||||
prompt = prompt.replace(VIDEO_PROMPT, video_string, 1)
|
prompt_text = prompt_text.replace(VIDEO_PROMPT, video_string, 1)
|
||||||
|
|
||||||
video_grids = video_outputs.pop("video_grids")
|
video_grids = video_outputs.pop("video_grids")
|
||||||
assert video_grids[:, 0].sum() == len(
|
assert video_grids[:, 0].sum() == len(
|
||||||
@@ -2054,10 +2059,10 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
|||||||
else:
|
else:
|
||||||
all_video_outputs = dict()
|
all_video_outputs = dict()
|
||||||
|
|
||||||
processed_outputs = self.info.ctx.call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
patched_call,
|
patched_call,
|
||||||
dict(text=prompt, **mm_data),
|
dict(text=prompt_text, **mm_data),
|
||||||
mm_kwargs,
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
if (images := mm_data.get("images")) is not None:
|
if (images := mm_data.get("images")) is not None:
|
||||||
@@ -2077,22 +2082,22 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
|||||||
for image_size in image_sizes
|
for image_size in image_sizes
|
||||||
]
|
]
|
||||||
num_crops = torch.tensor(tilings).prod(-1) + 1
|
num_crops = torch.tensor(tilings).prod(-1) + 1
|
||||||
assert sum(num_crops) == len(processed_outputs["pixel_values"])
|
assert sum(num_crops) == len(processed_data["pixel_values"])
|
||||||
assert sum(num_crops) == processed_outputs["image_num_crops"].sum().item()
|
assert sum(num_crops) == processed_data["image_num_crops"].sum().item()
|
||||||
|
|
||||||
image_grids = processed_outputs.pop("image_grids")
|
image_grids = processed_data.pop("image_grids")
|
||||||
image_num_pooled_patches = image_grids[:, :2].prod(dim=1) + image_grids[
|
image_num_pooled_patches = image_grids[:, :2].prod(dim=1) + image_grids[
|
||||||
:, 2:
|
:, 2:
|
||||||
].prod(dim=1)
|
].prod(dim=1)
|
||||||
|
|
||||||
processed_outputs["image_num_pooled_patches"] = image_num_pooled_patches
|
processed_data["image_num_pooled_patches"] = image_num_pooled_patches
|
||||||
n_patches = processed_outputs["pixel_values"].shape[1]
|
n_patches = processed_data["pixel_values"].shape[1]
|
||||||
processed_outputs["image_num_patches"] = (
|
processed_data["image_num_patches"] = (
|
||||||
processed_outputs["image_num_crops"] * n_patches
|
processed_data["image_num_crops"] * n_patches
|
||||||
)
|
)
|
||||||
(
|
(
|
||||||
processed_outputs["image_tokens"],
|
processed_data["image_tokens"],
|
||||||
processed_outputs["num_image_tokens"],
|
processed_data["num_image_tokens"],
|
||||||
) = build_flat_image_bool_length(
|
) = build_flat_image_bool_length(
|
||||||
image_grids,
|
image_grids,
|
||||||
hf_config,
|
hf_config,
|
||||||
@@ -2101,7 +2106,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
|||||||
use_single_crop_start_token=hf_processor.use_single_crop_start_token,
|
use_single_crop_start_token=hf_processor.use_single_crop_start_token,
|
||||||
)
|
)
|
||||||
|
|
||||||
return BatchFeature({**processed_outputs, **all_video_outputs})
|
processed_data.update(all_video_outputs)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
processed_data.pop("input_ids")
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -942,16 +942,6 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
|
|||||||
image_placeholder: str = "<image>"
|
image_placeholder: str = "<image>"
|
||||||
bos_image_placeholder: str = "<|endoftext|><image>"
|
bos_image_placeholder: str = "<|endoftext|><image>"
|
||||||
|
|
||||||
def _call_hf_processor(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
# Moondream3's processor handles images directly rather than exposing a
|
|
||||||
# separate `image_processor`, so keep the cache path on text+MM calls.
|
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def bos_image_placeholder_tokens(self) -> list[int]:
|
def bos_image_placeholder_tokens(self) -> list[int]:
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
|
|||||||
@@ -1335,23 +1335,32 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo
|
|||||||
|
|
||||||
|
|
||||||
class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]):
|
class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
mm_data = dict(mm_data)
|
mm_data = dict(mm_data)
|
||||||
audios = mm_data.pop("audios", [])
|
audios = mm_data.pop("audios", [])
|
||||||
if audios:
|
if audios:
|
||||||
mm_data["audio"] = audios
|
mm_data["audio"] = audios
|
||||||
mm_kwargs = dict(mm_kwargs)
|
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||||
processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs)
|
processor_kwargs = _filter_moss_audio_processor_config(hf_processor_mm_kwargs)
|
||||||
return self.info.ctx.call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
self.info.get_hf_processor(**processor_kwargs),
|
self.info.get_hf_processor(**processor_kwargs),
|
||||||
dict(text=prompt, **mm_data),
|
mm_data,
|
||||||
{},
|
{},
|
||||||
)
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -461,24 +461,29 @@ class MossTranscribeDiarizeDummyInputsBuilder(
|
|||||||
class MossTranscribeDiarizeMultiModalProcessor(
|
class MossTranscribeDiarizeMultiModalProcessor(
|
||||||
BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]
|
BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
tokenizer = self.info.get_tokenizer()
|
|
||||||
audios = _get_audios_from_mm_data(mm_data)
|
|
||||||
if not audios:
|
|
||||||
input_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
|
||||||
return BatchFeature({"input_ids": [input_ids]}, tensor_type="pt")
|
|
||||||
|
|
||||||
processed = self.info.ctx.call_hf_processor(
|
def _preprocess_hf_mm_data(
|
||||||
self.info.get_hf_processor(**mm_kwargs),
|
self,
|
||||||
dict(text=prompt, audio=audios),
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs,
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
)
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
return _add_vllm_audio_metadata(processed, len(audios))
|
audios = _get_audios_from_mm_data(mm_data)
|
||||||
|
|
||||||
|
return dict(audio=audios), hf_processor_mm_kwargs
|
||||||
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
audios = _get_audios_from_mm_data(mm_data)
|
||||||
|
if audios:
|
||||||
|
processed_data = _add_vllm_audio_metadata(processed_data, len(audios))
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -262,13 +262,19 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
|
|||||||
class MuseGlimmerMultiModalProcessor(
|
class MuseGlimmerMultiModalProcessor(
|
||||||
BaseMultiModalProcessor[MuseGlimmerProcessingInfo]
|
BaseMultiModalProcessor[MuseGlimmerProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
|
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
tokenizer = processor.tokenizer
|
tokenizer = processor.tokenizer
|
||||||
config = self.info.get_hf_config()
|
config = self.info.get_hf_config()
|
||||||
images = mm_data.get("images", ())
|
images = mm_data.get("images", ())
|
||||||
@@ -276,7 +282,7 @@ class MuseGlimmerMultiModalProcessor(
|
|||||||
if not isinstance(images, Sequence) or not isinstance(videos, Sequence):
|
if not isinstance(images, Sequence) or not isinstance(videos, Sequence):
|
||||||
raise TypeError("MuseGlimmer multi-modal data must be a sequence")
|
raise TypeError("MuseGlimmer multi-modal data must be a sequence")
|
||||||
|
|
||||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
prompt_ids = tokenizer.encode(prompt_text, add_special_tokens=False)
|
||||||
image_sentinel_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)
|
image_sentinel_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)
|
||||||
video_sentinel_id = tokenizer.convert_tokens_to_ids(VIDEO_TOKEN)
|
video_sentinel_id = tokenizer.convert_tokens_to_ids(VIDEO_TOKEN)
|
||||||
if prompt_ids.count(image_sentinel_id) != len(images):
|
if prompt_ids.count(image_sentinel_id) != len(images):
|
||||||
@@ -354,7 +360,9 @@ class MuseGlimmerMultiModalProcessor(
|
|||||||
video_pixel_values=video_pixels,
|
video_pixel_values=video_pixels,
|
||||||
video_feature_sizes=torch.tensor(video_sizes),
|
video_feature_sizes=torch.tensor(video_sizes),
|
||||||
)
|
)
|
||||||
return BatchFeature(data=data, tensor_type=None)
|
processed_data = BatchFeature(data=data, tensor_type=None)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -361,18 +361,8 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
|
|||||||
class NanoNemotronVLMultiModalProcessor(
|
class NanoNemotronVLMultiModalProcessor(
|
||||||
BaseMultiModalProcessor[NanoNemotronVLProcessingInfo]
|
BaseMultiModalProcessor[NanoNemotronVLProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
"""
|
|
||||||
Bypass `call_hf_processor_mm_only` by no-op overriding`_call_hf_processor`,
|
|
||||||
so it chooses this path:
|
|
||||||
`type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor`
|
|
||||||
"""
|
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
|
|
||||||
def _get_image_fields_config(self, hf_inputs: BatchFeature):
|
def _get_image_fields_config(self, hf_inputs: BatchFeature):
|
||||||
if self.info.is_dynamic_tiler:
|
if self.info.is_dynamic_tiler:
|
||||||
@@ -406,7 +396,9 @@ class NanoNemotronVLMultiModalProcessor(
|
|||||||
)
|
)
|
||||||
|
|
||||||
def _get_audio_fields_config(self, hf_inputs: BatchFeature):
|
def _get_audio_fields_config(self, hf_inputs: BatchFeature):
|
||||||
audio_num_clips = torch.as_tensor(hf_inputs["audio_num_clips"])
|
audio_num_clips = torch.as_tensor(
|
||||||
|
hf_inputs.get("audio_num_clips", torch.empty(0))
|
||||||
|
)
|
||||||
|
|
||||||
return dict(
|
return dict(
|
||||||
input_audio_features=MultiModalFieldConfig.flat_from_sizes(
|
input_audio_features=MultiModalFieldConfig.flat_from_sizes(
|
||||||
@@ -733,11 +725,8 @@ class NanoNemotronVLMultiModalProcessor(
|
|||||||
# Bypass the cached path: the HF processor must receive the
|
# Bypass the cached path: the HF processor must receive the
|
||||||
# prompt (with injected <so_embedding>) and the audio data
|
# prompt (with injected <so_embedding>) and the audio data
|
||||||
# together so it can perform audio-token replacement natively.
|
# together so it can perform audio-token replacement natively.
|
||||||
(
|
mm_info = self._apply_hf_processor(inputs, timing_ctx)
|
||||||
prompt_ids,
|
prompt_ids = self._postprocess_prompt(inputs.prompt)
|
||||||
mm_info,
|
|
||||||
is_update_applied,
|
|
||||||
) = self._apply_hf_processor(inputs, timing_ctx)
|
|
||||||
|
|
||||||
with timing_ctx.record("apply_prompt_updates"):
|
with timing_ctx.record("apply_prompt_updates"):
|
||||||
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
|
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
|
||||||
@@ -745,7 +734,7 @@ class NanoNemotronVLMultiModalProcessor(
|
|||||||
prompt_ids=prompt_ids,
|
prompt_ids=prompt_ids,
|
||||||
mm_kwargs=mm_info.kwargs,
|
mm_kwargs=mm_info.kwargs,
|
||||||
mm_prompt_updates=mm_info.prompt_updates,
|
mm_prompt_updates=mm_info.prompt_updates,
|
||||||
is_update_applied=is_update_applied,
|
is_update_applied=self.hf_processor_applies_updates,
|
||||||
)
|
)
|
||||||
|
|
||||||
mm_placeholder_ranges = {
|
mm_placeholder_ranges = {
|
||||||
|
|||||||
@@ -411,22 +411,6 @@ class NemotronParseMultiModalProcessor(
|
|||||||
) -> list[int]:
|
) -> list[int]:
|
||||||
return [0]
|
return [0]
|
||||||
|
|
||||||
def _call_hf_processor(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
if mm_data:
|
|
||||||
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
else:
|
|
||||||
hf_processor = self.info.get_hf_processor()
|
|
||||||
tokenizer = hf_processor.tokenizer
|
|
||||||
processed_outputs = tokenizer(
|
|
||||||
prompt, add_special_tokens=False, return_tensors="pt"
|
|
||||||
)
|
|
||||||
return processed_outputs
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
hf_inputs: BatchFeature,
|
hf_inputs: BatchFeature,
|
||||||
|
|||||||
@@ -341,41 +341,30 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
|
|||||||
# -300 is image_atom token, filter them out
|
# -300 is image_atom token, filter them out
|
||||||
return [vte_vocab_size + x + 300 for x in image_indicators if x < -300]
|
return [vte_vocab_size + x + 300 for x in image_indicators if x < -300]
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
if not mm_data:
|
if not mm_data:
|
||||||
# Avoid warning from HF logger for text-only input
|
return processed_data
|
||||||
tokenizer = self.info.get_tokenizer()
|
|
||||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
hf_processor = self.info.get_hf_processor()
|
hf_processor = self.info.get_hf_processor()
|
||||||
image_indicators = [
|
image_indicators = [
|
||||||
hf_processor.construct_image_indicators(grid)
|
hf_processor.construct_image_indicators(grid)
|
||||||
for grid in processed_outputs["grids"]
|
for grid in processed_data["grids"]
|
||||||
]
|
]
|
||||||
indicator_tokens = [
|
indicator_tokens = [
|
||||||
self.image_indicators_to_visual_tokens(indicator)
|
self.image_indicators_to_visual_tokens(indicator)
|
||||||
for indicator in image_indicators
|
for indicator in image_indicators
|
||||||
]
|
]
|
||||||
processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens)
|
processed_data["indicator_tokens"] = torch.tensor(indicator_tokens)
|
||||||
return processed_outputs
|
|
||||||
|
|
||||||
def _apply_hf_processor_tokens_only(
|
return processed_data
|
||||||
self,
|
|
||||||
prompt_tokens: list[int],
|
|
||||||
) -> list[int]:
|
|
||||||
return prompt_tokens
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -332,53 +332,43 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
|
|||||||
if x >= INDICATOR_IDS[0]
|
if x >= INDICATOR_IDS[0]
|
||||||
]
|
]
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
if not mm_data:
|
if not mm_data:
|
||||||
# Avoid warning from HF logger for text-only input
|
return processed_data
|
||||||
tokenizer = self.info.get_tokenizer()
|
|
||||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
hf_processor = self.info.get_hf_processor()
|
hf_processor = self.info.get_hf_processor()
|
||||||
|
|
||||||
if "videos" in mm_data:
|
if "videos" in mm_data:
|
||||||
visual_indicators = [
|
visual_indicators = [
|
||||||
hf_processor.construct_visual_indicators((1, 1, 1), True)
|
hf_processor.construct_visual_indicators((1, 1, 1), True)
|
||||||
for grid in processed_outputs["video_grids"]
|
for grid in processed_data["video_grids"]
|
||||||
]
|
]
|
||||||
indicator_tokens = [
|
indicator_tokens = [
|
||||||
self.visual_indicators_to_visual_tokens(indicator)
|
self.visual_indicators_to_visual_tokens(indicator)
|
||||||
for indicator in visual_indicators
|
for indicator in visual_indicators
|
||||||
]
|
]
|
||||||
processed_outputs["video_indicator_tokens"] = torch.tensor(indicator_tokens)
|
processed_data["video_indicator_tokens"] = torch.tensor(indicator_tokens)
|
||||||
if "images" in mm_data:
|
if "images" in mm_data:
|
||||||
visual_indicators = [
|
visual_indicators = [
|
||||||
hf_processor.construct_visual_indicators((1, 1, 1), False)
|
hf_processor.construct_visual_indicators((1, 1, 1), False)
|
||||||
for grid in processed_outputs["grids"]
|
for grid in processed_data["grids"]
|
||||||
]
|
]
|
||||||
indicator_tokens = [
|
indicator_tokens = [
|
||||||
self.visual_indicators_to_visual_tokens(indicator)
|
self.visual_indicators_to_visual_tokens(indicator)
|
||||||
for indicator in visual_indicators
|
for indicator in visual_indicators
|
||||||
]
|
]
|
||||||
|
|
||||||
processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens)
|
processed_data["indicator_tokens"] = torch.tensor(indicator_tokens)
|
||||||
return processed_outputs
|
|
||||||
|
|
||||||
def _apply_hf_processor_tokens_only(
|
return processed_data
|
||||||
self,
|
|
||||||
prompt_tokens: list[int],
|
|
||||||
) -> list[int]:
|
|
||||||
return prompt_tokens
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -244,32 +244,37 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing
|
|||||||
class PaddleOCRVLMultiModalProcessor(
|
class PaddleOCRVLMultiModalProcessor(
|
||||||
BaseMultiModalProcessor[PaddleOCRVLProcessingInfo]
|
BaseMultiModalProcessor[PaddleOCRVLProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
if mm_data:
|
valid_mm_items = mm_items.select(
|
||||||
final_mm_kwargs = dict(mm_kwargs or {})
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
final_mm_kwargs.setdefault("images_kwargs", {})
|
)
|
||||||
# vLLM use PIL.Image, always set channel_last
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
|
|
||||||
processed_outputs = self.info.ctx.call_hf_processor(
|
if not mm_data:
|
||||||
self.info.get_hf_processor(**final_mm_kwargs),
|
return BatchFeature(dict(passthrough_data))
|
||||||
dict(text=prompt, **mm_data),
|
|
||||||
mm_kwargs,
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
)
|
|
||||||
num_patches_per_image = processed_outputs["image_grid_thw"].prod(-1)
|
final_mm_kwargs = dict(hf_processor_mm_kwargs or {})
|
||||||
processed_outputs["pixel_values"] = processed_outputs["pixel_values"].split(
|
final_mm_kwargs.setdefault("images_kwargs", {})
|
||||||
num_patches_per_image.tolist()
|
# vLLM use PIL.Image, always set channel_last
|
||||||
)
|
final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
|
||||||
else:
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
tokenizer = self.info.get_tokenizer()
|
self.info.get_hf_processor(**final_mm_kwargs),
|
||||||
processed_outputs = tokenizer(
|
dict(text=prompt_text, **mm_data),
|
||||||
prompt, add_special_tokens=True, return_tensors="pt"
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
return processed_outputs
|
num_patches_per_image = processed_data["image_grid_thw"].prod(-1)
|
||||||
|
processed_data["pixel_values"] = processed_data["pixel_values"].split(
|
||||||
|
num_patches_per_image.tolist()
|
||||||
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -152,23 +152,6 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo
|
|||||||
|
|
||||||
|
|
||||||
class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingInfo]):
|
class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingInfo]):
|
||||||
def _call_hf_processor(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
tokenizer = self.info.get_tokenizer()
|
|
||||||
if not mm_data:
|
|
||||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
|
||||||
return super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
hf_inputs: BatchFeature,
|
hf_inputs: BatchFeature,
|
||||||
|
|||||||
@@ -396,19 +396,19 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
processed_outputs = super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
input_ids = processed_outputs["input_ids"]
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if not mm_data:
|
||||||
|
return processed_data
|
||||||
|
|
||||||
|
input_ids = processed_data["input_ids"]
|
||||||
assert isinstance(input_ids, torch.Tensor)
|
assert isinstance(input_ids, torch.Tensor)
|
||||||
|
|
||||||
# Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
|
# Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
|
||||||
@@ -416,7 +416,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
|||||||
# Therefore, we need to do an early replacement here
|
# Therefore, we need to do an early replacement here
|
||||||
input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID)
|
input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID)
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -868,43 +868,52 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
if not mm_data:
|
valid_mm_items = mm_items.select(
|
||||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
)
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
sr = self.info.get_feature_extractor(**mm_kwargs).sampling_rate
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
|
sr = self.info.get_feature_extractor(**hf_processor_mm_kwargs).sampling_rate
|
||||||
if audio_data := mm_data.get("audios", []):
|
if audio_data := mm_data.get("audios", []):
|
||||||
mm_data["audios"] = [(data, sr) for data in audio_data]
|
mm_data["audios"] = [(data, sr) for data in audio_data]
|
||||||
|
|
||||||
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
|
dict(text=prompt_text, **mm_data),
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
num_img_tokens = [
|
num_img_tokens = [
|
||||||
self.info.get_num_image_tokens(
|
self.info.get_num_image_tokens(
|
||||||
image_width=img_size[0],
|
image_width=img_size[0],
|
||||||
image_height=img_size[1],
|
image_height=img_size[1],
|
||||||
processor=hf_processor,
|
processor=hf_processor,
|
||||||
)
|
)
|
||||||
for img_size in processed_outputs["image_sizes"]
|
for img_size in processed_data["image_sizes"]
|
||||||
]
|
]
|
||||||
processed_outputs["num_img_tokens"] = num_img_tokens
|
processed_data["num_img_tokens"] = num_img_tokens
|
||||||
|
|
||||||
audio_features = processed_outputs["input_audio_embeds"]
|
audio_features = processed_data["input_audio_embeds"]
|
||||||
feature_sizes = [
|
feature_sizes = [
|
||||||
self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data
|
self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data
|
||||||
]
|
]
|
||||||
processed_outputs["input_audio_embeds"] = [
|
processed_data["input_audio_embeds"] = [
|
||||||
audio_features[idx, :size] for idx, size in enumerate(feature_sizes)
|
audio_features[idx, :size] for idx, size in enumerate(feature_sizes)
|
||||||
]
|
]
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
|
||||||
return processed_outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -142,16 +142,22 @@ class Phi4SiglipDummyInputsBuilder(
|
|||||||
class Phi4SiglipMultiModalProcessor(
|
class Phi4SiglipMultiModalProcessor(
|
||||||
BaseMultiModalProcessor[Phi4SiglipProcessingInfo],
|
BaseMultiModalProcessor[Phi4SiglipProcessingInfo],
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
processed = super()._call_hf_processor(
|
valid_mm_items = mm_items.select(
|
||||||
prompt=prompt,
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
mm_data=mm_data,
|
)
|
||||||
mm_kwargs=mm_kwargs,
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
|
processed = self.info.ctx.call_hf_processor(
|
||||||
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
|
dict(text=prompt_text, **mm_data),
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
# The HF processor's tokenizer_image_token() replaces the "<image>"
|
# The HF processor's tokenizer_image_token() replaces the "<image>"
|
||||||
@@ -163,10 +169,12 @@ class Phi4SiglipMultiModalProcessor(
|
|||||||
# NOTE: tokenizer.__call__() (not .encode()) must be used so that
|
# NOTE: tokenizer.__call__() (not .encode()) must be used so that
|
||||||
# added/special tokens like <|user|>, <|end|> are kept as single IDs.
|
# added/special tokens like <|user|>, <|end|> are kept as single IDs.
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
new_ids = tokenizer(prompt).input_ids
|
new_ids = tokenizer(prompt_text).input_ids
|
||||||
processed["input_ids"] = torch.tensor([new_ids])
|
processed["input_ids"] = torch.tensor([new_ids])
|
||||||
|
|
||||||
return processed
|
processed_data = processed
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -51,12 +51,10 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
|
|||||||
from vllm.multimodal.processing.processor import (
|
from vllm.multimodal.processing.processor import (
|
||||||
BaseMultiModalProcessor,
|
BaseMultiModalProcessor,
|
||||||
BaseProcessingInfo,
|
BaseProcessingInfo,
|
||||||
MultiModalProcessingInfo,
|
|
||||||
ProcessorInputs,
|
ProcessorInputs,
|
||||||
PromptReplacement,
|
PromptReplacement,
|
||||||
PromptUpdate,
|
PromptUpdate,
|
||||||
PromptUpdateDetails,
|
PromptUpdateDetails,
|
||||||
TimingContext,
|
|
||||||
)
|
)
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
@@ -213,6 +211,9 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]):
|
class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]):
|
||||||
|
# The tokens are already inserted by the chat template
|
||||||
|
hf_processor_applies_updates = True
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
hf_inputs: Mapping[str, NestedTensors],
|
hf_inputs: Mapping[str, NestedTensors],
|
||||||
@@ -220,24 +221,35 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
|
|||||||
) -> Mapping[str, MultiModalFieldConfig]:
|
) -> Mapping[str, MultiModalFieldConfig]:
|
||||||
return dict(images=MultiModalFieldConfig.batched("image"))
|
return dict(images=MultiModalFieldConfig.batched("image"))
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
outputs = self.info.ctx.call_hf_processor(
|
outputs = self.info.ctx.call_hf_processor(
|
||||||
self.info.get_hf_processor(**mm_kwargs),
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
dict(text=prompt, **mm_data),
|
dict(text=prompt_text, **mm_data),
|
||||||
# Avoid padding issue
|
# Avoid padding issue
|
||||||
dict(**mm_kwargs, return_tensors=None),
|
dict(**hf_processor_mm_kwargs, return_tensors=None),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Missing batch dimension
|
# Missing batch dimension
|
||||||
if is_list_of(outputs["input_ids"], int):
|
if is_list_of(outputs["input_ids"], int):
|
||||||
outputs["input_ids"] = [outputs["input_ids"]]
|
outputs["input_ids"] = [outputs["input_ids"]]
|
||||||
|
|
||||||
return outputs
|
processed_data = outputs
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
@@ -273,16 +285,6 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
|
|||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
def _cached_apply_hf_processor(
|
|
||||||
self,
|
|
||||||
inputs: ProcessorInputs,
|
|
||||||
timing_ctx: TimingContext,
|
|
||||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
|
||||||
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx)
|
|
||||||
|
|
||||||
# NOTE: The tokens are already inserted by the chat template
|
|
||||||
return prompt_ids, mm_info, True
|
|
||||||
|
|
||||||
|
|
||||||
@MULTIMODAL_REGISTRY.register_processor(
|
@MULTIMODAL_REGISTRY.register_processor(
|
||||||
PixtralMultiModalProcessor,
|
PixtralMultiModalProcessor,
|
||||||
|
|||||||
@@ -499,83 +499,6 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
|
|||||||
class Qwen2_5OmniThinkerMultiModalProcessor(
|
class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||||
BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
|
BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
mm_data = dict(mm_data)
|
|
||||||
audios = mm_data.pop("audios", [])
|
|
||||||
|
|
||||||
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
|
|
||||||
if audios:
|
|
||||||
# NOTE: Qwen2.5-Omni processor accept "audio"
|
|
||||||
mm_data["audio"] = audios
|
|
||||||
mm_kwargs = dict(
|
|
||||||
**mm_kwargs,
|
|
||||||
)
|
|
||||||
elif (
|
|
||||||
mm_kwargs.get("use_audio_in_video")
|
|
||||||
and "audio" not in mm_data
|
|
||||||
and mm_data.get("videos")
|
|
||||||
):
|
|
||||||
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
|
|
||||||
# and populated mm_data["audio"] itself before delegating here,
|
|
||||||
# in which case this isn't a genuine "no audio" case. Likewise,
|
|
||||||
# mm_data can be empty on a multimodal-processor-cache hit, where
|
|
||||||
# there's nothing to (re-)process this call and the real result
|
|
||||||
# comes from the cache — not a genuine "no audio" case either.
|
|
||||||
raise ValueError(
|
|
||||||
"Video doesn't have audio track with `audio_in_video=True`"
|
|
||||||
)
|
|
||||||
|
|
||||||
merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
|
|
||||||
if mm_data.get("videos") and (
|
|
||||||
merged.keys() & {"size", "min_pixels", "max_pixels"}
|
|
||||||
):
|
|
||||||
mm_kwargs = dict(mm_kwargs)
|
|
||||||
video_size = dict(self.info.get_hf_processor().video_processor.size)
|
|
||||||
size_override = merged.get("size")
|
|
||||||
if size_override is not None:
|
|
||||||
video_size = video_size | size_override
|
|
||||||
min_pixels = merged.get("min_pixels")
|
|
||||||
if min_pixels is not None:
|
|
||||||
video_size["shortest_edge"] = min_pixels
|
|
||||||
max_pixels = merged.get("max_pixels")
|
|
||||||
if max_pixels is not None:
|
|
||||||
video_size["longest_edge"] = max_pixels
|
|
||||||
mm_kwargs["size"] = video_size
|
|
||||||
|
|
||||||
hf_inputs = super()._call_hf_processor(
|
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
input_features = hf_inputs.pop("input_features", None)
|
|
||||||
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
|
|
||||||
if "input_audio_features" not in hf_inputs and input_features is not None:
|
|
||||||
if feature_attention_mask is not None:
|
|
||||||
input_features = input_features.permute(0, 2, 1)[
|
|
||||||
feature_attention_mask.bool()
|
|
||||||
].permute(1, 0)
|
|
||||||
hf_inputs["input_audio_features"] = input_features
|
|
||||||
if (
|
|
||||||
"audio_feature_lengths" not in hf_inputs
|
|
||||||
and feature_attention_mask is not None
|
|
||||||
):
|
|
||||||
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
|
|
||||||
|
|
||||||
video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
|
|
||||||
if video_second_per_grid is not None:
|
|
||||||
hf_inputs["second_per_grid_ts"] = video_second_per_grid
|
|
||||||
|
|
||||||
use_audio_in_video = mm_kwargs.get("use_audio_in_video", False)
|
|
||||||
hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
|
|
||||||
|
|
||||||
return hf_inputs
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
hf_inputs: BatchFeature,
|
hf_inputs: BatchFeature,
|
||||||
@@ -885,7 +808,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
|||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
def _apply_hf_processor_mm_only(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
mm_items: MultiModalDataItems,
|
mm_items: MultiModalDataItems,
|
||||||
hf_processor_mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
@@ -903,13 +826,87 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
|||||||
)
|
)
|
||||||
mm_counts["audio"] -= mm_counts["video"]
|
mm_counts["audio"] -= mm_counts["video"]
|
||||||
|
|
||||||
_, mm_processed_data = self._apply_hf_processor_text_mm(
|
valid_mm_items = mm_items.select(
|
||||||
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
mm_items=mm_items,
|
)
|
||||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
mm_data = dict(processor_data)
|
||||||
|
audios = mm_data.pop("audios", [])
|
||||||
|
|
||||||
|
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
|
||||||
|
if audios:
|
||||||
|
# NOTE: Qwen2.5-Omni processor accept "audio"
|
||||||
|
mm_data["audio"] = audios
|
||||||
|
hf_processor_mm_kwargs = dict(
|
||||||
|
**hf_processor_mm_kwargs,
|
||||||
|
)
|
||||||
|
elif (
|
||||||
|
hf_processor_mm_kwargs.get("use_audio_in_video")
|
||||||
|
and "audio" not in mm_data
|
||||||
|
and mm_data.get("videos")
|
||||||
|
):
|
||||||
|
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
|
||||||
|
# and populated mm_data["audio"] itself before delegating here,
|
||||||
|
# in which case this isn't a genuine "no audio" case. Likewise,
|
||||||
|
# mm_data can be empty on a multimodal-processor-cache hit, where
|
||||||
|
# there's nothing to (re-)process this call and the real result
|
||||||
|
# comes from the cache — not a genuine "no audio" case either.
|
||||||
|
raise ValueError(
|
||||||
|
"Video doesn't have audio track with `audio_in_video=True`"
|
||||||
|
)
|
||||||
|
|
||||||
|
merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||||
|
if mm_data.get("videos") and (
|
||||||
|
merged.keys() & {"size", "min_pixels", "max_pixels"}
|
||||||
|
):
|
||||||
|
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||||
|
video_size = dict(self.info.get_hf_processor().video_processor.size)
|
||||||
|
size_override = merged.get("size")
|
||||||
|
if size_override is not None:
|
||||||
|
video_size = video_size | size_override
|
||||||
|
min_pixels = merged.get("min_pixels")
|
||||||
|
if min_pixels is not None:
|
||||||
|
video_size["shortest_edge"] = min_pixels
|
||||||
|
max_pixels = merged.get("max_pixels")
|
||||||
|
if max_pixels is not None:
|
||||||
|
video_size["longest_edge"] = max_pixels
|
||||||
|
hf_processor_mm_kwargs["size"] = video_size
|
||||||
|
|
||||||
|
hf_inputs = self.info.ctx.call_hf_processor(
|
||||||
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
|
dict(text=prompt_text, **mm_data),
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
return mm_processed_data
|
input_features = hf_inputs.pop("input_features", None)
|
||||||
|
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
|
||||||
|
if "input_audio_features" not in hf_inputs and input_features is not None:
|
||||||
|
if feature_attention_mask is not None:
|
||||||
|
input_features = input_features.permute(0, 2, 1)[
|
||||||
|
feature_attention_mask.bool()
|
||||||
|
].permute(1, 0)
|
||||||
|
hf_inputs["input_audio_features"] = input_features
|
||||||
|
if (
|
||||||
|
"audio_feature_lengths" not in hf_inputs
|
||||||
|
and feature_attention_mask is not None
|
||||||
|
):
|
||||||
|
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
|
||||||
|
|
||||||
|
video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
|
||||||
|
if video_second_per_grid is not None:
|
||||||
|
hf_inputs["second_per_grid_ts"] = video_second_per_grid
|
||||||
|
|
||||||
|
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
|
||||||
|
hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
|
||||||
|
|
||||||
|
processed_data = hf_inputs
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
processed_data.pop("input_ids")
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
|
|
||||||
class Qwen2_5OmniConditionalGenerationMixin:
|
class Qwen2_5OmniConditionalGenerationMixin:
|
||||||
|
|||||||
@@ -1154,16 +1154,6 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
|||||||
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _call_hf_processor(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
# Override to use the text path instead of token path to use the
|
|
||||||
# video-specific logic in processing_qwen2_5_vl.py
|
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
mm_items: MultiModalDataItems,
|
mm_items: MultiModalDataItems,
|
||||||
|
|||||||
@@ -235,36 +235,27 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
|
|||||||
|
|
||||||
|
|
||||||
class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
|
class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, Any],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
) -> BatchFeature:
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
# NOTE - we rename audios -> audio in mm data because transformers has
|
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||||
# deprecated audios for the qwen2audio processor and will remove
|
|
||||||
# support for it in transformers 4.54.
|
mm_data = dict(mm_data)
|
||||||
audios = mm_data.pop("audios", [])
|
audios = mm_data.pop("audios", [])
|
||||||
if audios:
|
if audios:
|
||||||
mm_data["audio"] = audios
|
mm_data["audio"] = audios
|
||||||
|
|
||||||
# Text-only input not supported in composite processor
|
hf_processor_mm_kwargs = dict(
|
||||||
if not mm_data.get("audio", []):
|
**hf_processor_mm_kwargs,
|
||||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
|
||||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
|
||||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
|
||||||
mm_kwargs = dict(
|
|
||||||
**mm_kwargs,
|
|
||||||
sampling_rate=feature_extractor.sampling_rate,
|
sampling_rate=feature_extractor.sampling_rate,
|
||||||
)
|
)
|
||||||
|
|
||||||
return super()._call_hf_processor(
|
return mm_data, hf_processor_mm_kwargs
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -1195,79 +1195,95 @@ Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder
|
|||||||
class Qwen3OmniMoeThinkerMultiModalProcessor(
|
class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||||
Qwen2_5OmniThinkerMultiModalProcessor,
|
Qwen2_5OmniThinkerMultiModalProcessor,
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _get_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
mm_kwargs: Mapping[str, object],
|
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
||||||
) -> BatchFeature:
|
|
||||||
mm_data = dict(mm_data)
|
|
||||||
audios = mm_data.pop("audios", [])
|
|
||||||
|
|
||||||
def pad_to_hop_length(x: np.ndarray, hop_length: int) -> np.ndarray:
|
audios = processor_data.get("audios")
|
||||||
length = x.shape[-1]
|
|
||||||
if length % hop_length != 0:
|
|
||||||
pad_length = hop_length - (length % hop_length)
|
|
||||||
x = np.pad(x, (0, pad_length), mode="constant", constant_values=0)
|
|
||||||
return x
|
|
||||||
|
|
||||||
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
|
|
||||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
|
||||||
hop_length = feature_extractor.hop_length
|
|
||||||
if audios:
|
if audios:
|
||||||
# NOTE: Qwen3-Omni processor accept "audio"
|
feature_extractor = self.info.get_feature_extractor()
|
||||||
|
hop_length = feature_extractor.hop_length
|
||||||
|
|
||||||
|
def pad_to_hop_length(x: np.ndarray) -> np.ndarray:
|
||||||
|
length = x.shape[-1]
|
||||||
|
if length % hop_length != 0:
|
||||||
|
pad_length = hop_length - (length % hop_length)
|
||||||
|
x = np.pad(x, (0, pad_length), mode="constant", constant_values=0)
|
||||||
|
return x
|
||||||
|
|
||||||
# To make sure the cache works with padding=True, we pre-padded
|
# To make sure the cache works with padding=True, we pre-padded
|
||||||
# the audio to multiple of hop_length.
|
# the audio to a multiple of hop_length.
|
||||||
mm_data["audio"] = [
|
processor_data = dict(processor_data)
|
||||||
pad_to_hop_length(audio, hop_length)
|
processor_data["audios"] = [
|
||||||
|
pad_to_hop_length(audio)
|
||||||
if isinstance(audio, np.ndarray)
|
if isinstance(audio, np.ndarray)
|
||||||
else (pad_to_hop_length(audio[0], hop_length), audio[1])
|
else (pad_to_hop_length(audio[0]), audio[1])
|
||||||
for audio in audios
|
for audio in audios
|
||||||
]
|
]
|
||||||
|
|
||||||
|
return processor_data, passthrough_data
|
||||||
|
|
||||||
|
def _apply_hf_processor_main(
|
||||||
|
self,
|
||||||
|
mm_items: MultiModalDataItems,
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
) -> BatchFeature:
|
||||||
|
if mm_items.get_all_counts().get("audio", 0):
|
||||||
# TODO(Isotr0py): Remove this patch after upstream fix PR
|
# TODO(Isotr0py): Remove this patch after upstream fix PR
|
||||||
# released and Transformers version update:
|
# released and Transformers version update:
|
||||||
# https://github.com/huggingface/transformers/pull/41473
|
# https://github.com/huggingface/transformers/pull/41473
|
||||||
mm_kwargs = dict(mm_kwargs)
|
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||||
mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {})
|
hf_processor_mm_kwargs["audio_kwargs"] = dict(
|
||||||
mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {})
|
hf_processor_mm_kwargs.get("audio_kwargs") or {}
|
||||||
elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"):
|
)
|
||||||
# mm_data can be empty on a multimodal-processor-cache hit, where
|
hf_processor_mm_kwargs["text_kwargs"] = dict(
|
||||||
# there's nothing to (re-)process this call and the real result
|
hf_processor_mm_kwargs.get("text_kwargs") or {}
|
||||||
# comes from the cache — not a genuine "no audio" case.
|
|
||||||
raise ValueError(
|
|
||||||
"Video doesn't have audio track with `audio_in_video=True`"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
hf_inputs = super()._call_hf_processor(
|
processed_data = super()._apply_hf_processor_main(
|
||||||
prompt=prompt,
|
mm_items,
|
||||||
mm_data=mm_data,
|
hf_processor_mm_kwargs,
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
"audio_feature_lengths" in hf_inputs
|
"audio_feature_lengths" in processed_data
|
||||||
and "feature_attention_mask" in hf_inputs
|
and "feature_attention_mask" in processed_data
|
||||||
and (audios := mm_data.get("audio", []))
|
|
||||||
):
|
):
|
||||||
audio_num_frames = []
|
valid_mm_items = mm_items.select(
|
||||||
for _, audio in enumerate(audios):
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
audio_length = len(audio[0]) if isinstance(audio, tuple) else len(audio)
|
)
|
||||||
num_frame = (
|
processor_data, _ = self._get_hf_mm_data(valid_mm_items)
|
||||||
(audio_length // hop_length)
|
audios = processor_data.get("audios", [])
|
||||||
if audio_length % hop_length == 0
|
if audios:
|
||||||
else (audio_length // hop_length - 1)
|
feature_extractor = self.info.get_feature_extractor(
|
||||||
|
**hf_processor_mm_kwargs
|
||||||
)
|
)
|
||||||
if mm_kwargs.get("truncation", False):
|
hop_length = feature_extractor.hop_length
|
||||||
num_frame = min(
|
|
||||||
num_frame, feature_extractor.n_samples // hop_length
|
audio_num_frames = []
|
||||||
|
for audio in audios:
|
||||||
|
audio_length = (
|
||||||
|
len(audio[0]) if isinstance(audio, tuple) else len(audio)
|
||||||
)
|
)
|
||||||
audio_num_frames.append(num_frame)
|
num_frame = (
|
||||||
hf_inputs["feature_attention_mask"] = [
|
(audio_length // hop_length)
|
||||||
torch.ones(num_frame) for num_frame in audio_num_frames
|
if audio_length % hop_length == 0
|
||||||
]
|
else (audio_length // hop_length - 1)
|
||||||
hf_inputs["audio_feature_lengths"] = torch.tensor(audio_num_frames)
|
)
|
||||||
return hf_inputs
|
if hf_processor_mm_kwargs.get("truncation", False):
|
||||||
|
num_frame = min(
|
||||||
|
num_frame, feature_extractor.n_samples // hop_length
|
||||||
|
)
|
||||||
|
audio_num_frames.append(num_frame)
|
||||||
|
|
||||||
|
processed_data["feature_attention_mask"] = [
|
||||||
|
torch.ones(num_frame) for num_frame in audio_num_frames
|
||||||
|
]
|
||||||
|
processed_data["audio_feature_lengths"] = torch.tensor(audio_num_frames)
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _maybe_apply_prompt_updates(
|
def _maybe_apply_prompt_updates(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -1280,12 +1280,21 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
|||||||
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
|
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
|
||||||
return proc_impl is not None and proc_impl is not mixin_impl
|
return proc_impl is not None and proc_impl is not mixin_impl
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
mm_data = dict(mm_data)
|
mm_data = dict(mm_data)
|
||||||
|
|
||||||
# Separate video processing from image processing. Because the videos
|
# Separate video processing from image processing. Because the videos
|
||||||
@@ -1314,8 +1323,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
|||||||
|
|
||||||
# NOTE: a copy of is created to update do_sample_frames,
|
# NOTE: a copy of is created to update do_sample_frames,
|
||||||
# otherwise mm_hash for the object will be incorrect.
|
# otherwise mm_hash for the object will be incorrect.
|
||||||
video_mm_kwargs = dict(**mm_kwargs)
|
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
|
||||||
merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
|
merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||||
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
|
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
|
||||||
video_size = dict(self.info.get_video_processor().size)
|
video_size = dict(self.info.get_video_processor().size)
|
||||||
size_override = merged.get("size")
|
size_override = merged.get("size")
|
||||||
@@ -1365,10 +1374,13 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
|||||||
if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
|
if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
|
||||||
video_mm_kwargs["fps"] = None
|
video_mm_kwargs["fps"] = None
|
||||||
|
|
||||||
video_outputs = super()._call_hf_processor(
|
video_outputs = self.info.ctx.call_hf_processor(
|
||||||
prompt="<|vision_start|><|video_pad|><|vision_end|>",
|
self.info.get_hf_processor(**video_mm_kwargs),
|
||||||
mm_data=video_mm_data,
|
dict(
|
||||||
mm_kwargs=video_mm_kwargs,
|
text="<|vision_start|><|video_pad|><|vision_end|>",
|
||||||
|
**video_mm_data,
|
||||||
|
),
|
||||||
|
video_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Discard HF output input_ids — we use get_video_repl below
|
# Discard HF output input_ids — we use get_video_repl below
|
||||||
@@ -1424,12 +1436,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
|||||||
# fps/num_frames are video-only kwargs already consumed by the loop;
|
# fps/num_frames are video-only kwargs already consumed by the loop;
|
||||||
# exclude them so the text/image processor call below never gets a list.
|
# exclude them so the text/image processor call below never gets a list.
|
||||||
non_video_mm_kwargs = {
|
non_video_mm_kwargs = {
|
||||||
k: v for k, v in mm_kwargs.items() if k not in ("fps", "num_frames")
|
k: v
|
||||||
|
for k, v in hf_processor_mm_kwargs.items()
|
||||||
|
if k not in ("fps", "num_frames")
|
||||||
}
|
}
|
||||||
processed_outputs = super()._call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
prompt=prompt,
|
self.info.get_hf_processor(**non_video_mm_kwargs),
|
||||||
mm_data=mm_data,
|
dict(text=prompt_text, **mm_data),
|
||||||
mm_kwargs=non_video_mm_kwargs,
|
non_video_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Replace each placeholder with pre-computed video tokens.
|
# Replace each placeholder with pre-computed video tokens.
|
||||||
@@ -1443,20 +1457,19 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
|||||||
hf_config.video_token_id,
|
hf_config.video_token_id,
|
||||||
hf_config.vision_end_token_id,
|
hf_config.vision_end_token_id,
|
||||||
]
|
]
|
||||||
input_ids = processed_outputs.pop("input_ids")
|
input_ids = processed_data.pop("input_ids")
|
||||||
if not isinstance(input_ids, list):
|
if not isinstance(input_ids, list):
|
||||||
input_ids = input_ids.tolist()
|
input_ids = input_ids.tolist()
|
||||||
(prompt_ids,) = input_ids
|
(prompt_ids,) = input_ids
|
||||||
expanded_ids = _replace_video_token_placeholders(
|
expanded_ids = _replace_video_token_placeholders(
|
||||||
prompt_ids, video_target, video_input_ids_lst
|
prompt_ids, video_target, video_input_ids_lst
|
||||||
)
|
)
|
||||||
processed_outputs["input_ids"] = [expanded_ids]
|
processed_data["input_ids"] = [expanded_ids]
|
||||||
|
|
||||||
combined_outputs = dict(
|
processed_data.update(video_outputs)
|
||||||
processed_outputs,
|
processed_data.update(passthrough_data)
|
||||||
**video_outputs,
|
|
||||||
)
|
return processed_data
|
||||||
return BatchFeature(combined_outputs)
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -345,28 +345,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
|||||||
|
|
||||||
return mm_fields
|
return mm_fields
|
||||||
|
|
||||||
def _call_hf_processor(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> "BatchFeature":
|
|
||||||
"""Run the HF processor and unpad inputs."""
|
|
||||||
try:
|
|
||||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
except ValueError:
|
|
||||||
if any(mm_data.values()):
|
|
||||||
raise
|
|
||||||
# Some processors reject a prompt holding placeholders with
|
|
||||||
# no data to go with them, so tokenize it without them
|
|
||||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
|
||||||
hf_inputs = transformers.BatchFeature(
|
|
||||||
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
|
||||||
)
|
|
||||||
self._unpad_images(hf_inputs)
|
|
||||||
self._unpad_audios(hf_inputs, mm_data, mm_kwargs)
|
|
||||||
return hf_inputs
|
|
||||||
|
|
||||||
def _unpad_audios(
|
def _unpad_audios(
|
||||||
self,
|
self,
|
||||||
hf_inputs: "BatchFeature",
|
hf_inputs: "BatchFeature",
|
||||||
@@ -450,19 +428,6 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
|||||||
deriving them from updates."""
|
deriving them from updates."""
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def _call_hf_processor(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> "BatchFeature":
|
|
||||||
"""Ask which modality owns each token of the expanded prompt, which is the
|
|
||||||
only marking of the tokens an expansion adds around an item."""
|
|
||||||
if any(mm_data.values()):
|
|
||||||
mm_data = {**mm_data, "return_mm_token_type_ids": True}
|
|
||||||
|
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
|
|
||||||
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
||||||
"""Token ids marking where `modality` sits in the prompt, which for some
|
"""Token ids marking where `modality` sits in the prompt, which for some
|
||||||
processors differ from the placeholder written into it.
|
processors differ from the placeholder written into it.
|
||||||
@@ -612,24 +577,55 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
|||||||
|
|
||||||
with timing_ctx.record("apply_hf_processor"):
|
with timing_ctx.record("apply_hf_processor"):
|
||||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
prompt = hf_processor.decode(prompt)
|
prompt_text = hf_processor.decode(prompt)
|
||||||
|
|
||||||
# Bypass cached processor and always apply to the full set of mm inputs
|
# Bypass cached processor and always apply to the full set of mm inputs
|
||||||
# NOTE: we can't just set caching=False because base class method
|
# NOTE: we can't just set caching=False because base class method
|
||||||
# transforms outputs to `MultiModalKwargs` which is not going to
|
# transforms outputs to `MultiModalKwargs` which is not going to
|
||||||
# work for Transformers. The vision path has logic tied to
|
# work for Transformers. The vision path has logic tied to
|
||||||
# `mm_tokens_per_modality` in _apply_vision()
|
# `mm_tokens_per_modality` in _apply_vision()
|
||||||
prompt_ids, processed_data = self._apply_hf_processor_text_mm(
|
hf_processor_mm_kwargs = {
|
||||||
prompt_text=prompt,
|
**hf_processor_mm_kwargs,
|
||||||
mm_items=mm_items,
|
# HF processors only accept text, and the decoded string already
|
||||||
hf_processor_mm_kwargs={
|
# contains any special tokens, so don't let them be added again
|
||||||
**hf_processor_mm_kwargs,
|
# (the HF processor call disables `add_special_tokens`).
|
||||||
# HF processors only accept text, and the decoded string already
|
"add_special_tokens": False,
|
||||||
# contains any special tokens, so don't let them be added again
|
}
|
||||||
# (`_call_hf_processor` disables `add_special_tokens`).
|
|
||||||
"add_special_tokens": False,
|
valid_mm_items = mm_items.select(
|
||||||
},
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
)
|
)
|
||||||
|
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
# Ask which modality owns each token of the expanded prompt, which
|
||||||
|
# is the only marking of the tokens an expansion adds around an item
|
||||||
|
if any(processor_data.values()):
|
||||||
|
processor_data = {**processor_data, "return_mm_token_type_ids": True}
|
||||||
|
|
||||||
|
try:
|
||||||
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
|
dict(text=prompt_text, **processor_data),
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
if any(processor_data.values()):
|
||||||
|
raise
|
||||||
|
# Some processors reject a prompt holding placeholders with
|
||||||
|
# no data to go with them, so tokenize it without them
|
||||||
|
prompt_ids = self.info.get_tokenizer().encode(prompt_text)
|
||||||
|
processed_data = transformers.BatchFeature(
|
||||||
|
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
||||||
|
)
|
||||||
|
self._unpad_images(processed_data)
|
||||||
|
self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
|
||||||
|
input_ids = processed_data.pop("input_ids")
|
||||||
|
if not isinstance(input_ids, list):
|
||||||
|
input_ids = input_ids.tolist()
|
||||||
|
|
||||||
|
(prompt_ids,) = input_ids
|
||||||
|
|
||||||
# Use overrides if provided; fallback to data-dependent hashing.
|
# Use overrides if provided; fallback to data-dependent hashing.
|
||||||
with timing_ctx.record("get_mm_hashes"):
|
with timing_ctx.record("get_mm_hashes"):
|
||||||
@@ -773,17 +769,41 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
|||||||
except (AttributeError, KeyError, TypeError):
|
except (AttributeError, KeyError, TypeError):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> "BatchFeature":
|
) -> "BatchFeature":
|
||||||
"""Ask for the replacement each placeholder expands to, and record it as
|
mm_counts = mm_items.get_all_counts()
|
||||||
per-item fields: its token ids, and the tokens or patches behind them."""
|
|
||||||
|
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
# Ask for the replacement each placeholder expands to, and record it as
|
||||||
|
# per-item fields: its token ids, and the tokens or patches behind them
|
||||||
if has_mm_data := any(mm_data.values()):
|
if has_mm_data := any(mm_data.values()):
|
||||||
mm_data = {**mm_data, "return_text_replacement_offsets": True}
|
mm_data = {**mm_data, "return_text_replacement_offsets": True}
|
||||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
|
try:
|
||||||
|
hf_inputs = self.info.ctx.call_hf_processor(
|
||||||
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
|
dict(text=prompt_text, **mm_data),
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
if any(mm_data.values()):
|
||||||
|
raise
|
||||||
|
# Some processors reject a prompt holding placeholders with
|
||||||
|
# no data to go with them, so tokenize it without them
|
||||||
|
tokenizer = self.info.get_tokenizer()
|
||||||
|
hf_inputs = transformers.BatchFeature(
|
||||||
|
dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
|
||||||
|
)
|
||||||
|
self._unpad_images(hf_inputs)
|
||||||
|
self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)
|
||||||
|
|
||||||
# Drop the inputs the model would reject
|
# Drop the inputs the model would reject
|
||||||
hf_inputs.pop("mm_token_type_ids", None)
|
hf_inputs.pop("mm_token_type_ids", None)
|
||||||
hf_inputs.pop("token_type_ids", None)
|
hf_inputs.pop("token_type_ids", None)
|
||||||
@@ -802,6 +822,8 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
|||||||
"so it can be fixed, and install `transformers<5.15.0` in the "
|
"so it can be fixed, and install `transformers<5.15.0` in the "
|
||||||
"meantime to locate placeholders in the expanded prompt instead."
|
"meantime to locate placeholders in the expanded prompt instead."
|
||||||
)
|
)
|
||||||
|
hf_inputs.update(passthrough_data)
|
||||||
|
hf_inputs.pop("input_ids", None)
|
||||||
return hf_inputs
|
return hf_inputs
|
||||||
|
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
@@ -829,6 +851,9 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
|||||||
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
|
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
|
||||||
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
|
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
|
||||||
|
|
||||||
|
hf_inputs.update(passthrough_data)
|
||||||
|
hf_inputs.pop("input_ids")
|
||||||
|
|
||||||
return hf_inputs
|
return hf_inputs
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -190,41 +190,84 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
|
|||||||
|
|
||||||
|
|
||||||
class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
|
class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
# Text-only input not supported in composite processor
|
|
||||||
if not mm_data.get("audios", []):
|
|
||||||
prompt_ids = self.info.get_tokenizer().encode(
|
|
||||||
prompt, add_special_tokens=False
|
|
||||||
)
|
|
||||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
|
||||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
|
||||||
|
|
||||||
mm_data = dict(mm_data)
|
def _preprocess_hf_mm_data(
|
||||||
audios = mm_data.pop("audios", [])
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
|
audios = mm_data.get("audios", [])
|
||||||
assert isinstance(audios, list)
|
assert isinstance(audios, list)
|
||||||
|
|
||||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||||
mm_kwargs = dict(
|
hf_processor_mm_kwargs = dict(
|
||||||
**mm_kwargs,
|
**hf_processor_mm_kwargs,
|
||||||
sampling_rate=feature_extractor.sampling_rate,
|
sampling_rate=feature_extractor.sampling_rate,
|
||||||
include_audio_num_chunks=True,
|
include_audio_num_chunks=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
item_processor_data = dict(**mm_data, audios=audios)
|
return mm_data, hf_processor_mm_kwargs
|
||||||
|
|
||||||
output = super()._call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
prompt=prompt,
|
self,
|
||||||
mm_data=item_processor_data,
|
mm_items: MultiModalDataItems,
|
||||||
mm_kwargs=mm_kwargs,
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
)
|
)
|
||||||
output["audio_features"] = output.pop("audio_values")
|
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
return output
|
if processor_data:
|
||||||
|
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
|
||||||
|
processor_data,
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
|
||||||
|
if prompt_text is not None:
|
||||||
|
processor_data = dict(text=prompt_text, **processor_data)
|
||||||
|
|
||||||
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
|
||||||
|
def patched_call(**kwargs) -> BatchFeature:
|
||||||
|
# The remote UltravoxProcessor does not support the
|
||||||
|
# `truncation` kwarg injected by `call_hf_processor`: it
|
||||||
|
# passes `truncation` to its audio processor by itself while
|
||||||
|
# also forwarding `**kwargs` there, causing a duplicate
|
||||||
|
# keyword argument error. Its hardcoded `truncation=False`
|
||||||
|
# already matches vLLM's intended behavior.
|
||||||
|
kwargs.pop("truncation", None)
|
||||||
|
|
||||||
|
return hf_processor(**kwargs)
|
||||||
|
|
||||||
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
|
patched_call,
|
||||||
|
processor_data,
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
else:
|
||||||
|
processed_data = BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
return self._postprocess_hf_mm_data(
|
||||||
|
processor_data,
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
|
processed_data,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if "audio_values" in processed_data:
|
||||||
|
processed_data["audio_features"] = processed_data.pop("audio_values")
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -225,7 +225,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):
|
|||||||
self,
|
self,
|
||||||
inputs: ProcessorInputs,
|
inputs: ProcessorInputs,
|
||||||
timing_ctx: TimingContext,
|
timing_ctx: TimingContext,
|
||||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
) -> MultiModalProcessingInfo:
|
||||||
# The processor logic differs for single-image (crop) vs multi-image
|
# The processor logic differs for single-image (crop) vs multi-image
|
||||||
# (no crop) requests. The processing cache assumes per-item output is
|
# (no crop) requests. The processing cache assumes per-item output is
|
||||||
# invariant of how many images are passed per prompt, so we only cache
|
# invariant of how many images are passed per prompt, so we only cache
|
||||||
|
|||||||
@@ -47,12 +47,10 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
|
|||||||
from vllm.multimodal.processing.processor import (
|
from vllm.multimodal.processing.processor import (
|
||||||
BaseMultiModalProcessor,
|
BaseMultiModalProcessor,
|
||||||
BaseProcessingInfo,
|
BaseProcessingInfo,
|
||||||
MultiModalProcessingInfo,
|
|
||||||
PlaceholderFeaturesInfo,
|
PlaceholderFeaturesInfo,
|
||||||
ProcessorInputs,
|
ProcessorInputs,
|
||||||
PromptReplacement,
|
PromptReplacement,
|
||||||
PromptUpdate,
|
PromptUpdate,
|
||||||
TimingContext,
|
|
||||||
)
|
)
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.tokenizers import cached_tokenizer_from_config
|
from vllm.tokenizers import cached_tokenizer_from_config
|
||||||
@@ -204,6 +202,9 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
|
class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
|
||||||
|
# The tokens are already inserted by the chat template
|
||||||
|
hf_processor_applies_updates = True
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
hf_inputs: Mapping[str, NestedTensors],
|
hf_inputs: Mapping[str, NestedTensors],
|
||||||
@@ -220,12 +221,14 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
|||||||
# skip validation here
|
# skip validation here
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
) -> BatchFeature:
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
mm_data = dict(mm_data)
|
mm_data = dict(mm_data)
|
||||||
audios = mm_data.pop("audios", [])
|
audios = mm_data.pop("audios", [])
|
||||||
|
|
||||||
@@ -233,18 +236,23 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
|||||||
# MistralCommonVoxtralProcessor accepts "audio"
|
# MistralCommonVoxtralProcessor accepts "audio"
|
||||||
mm_data["audio"] = audios
|
mm_data["audio"] = audios
|
||||||
|
|
||||||
outputs = self.info.ctx.call_hf_processor(
|
# Avoid padding issue
|
||||||
self.info.get_hf_processor(**mm_kwargs),
|
hf_processor_mm_kwargs = dict(**hf_processor_mm_kwargs, return_tensors=None)
|
||||||
dict(text=prompt, **mm_data),
|
|
||||||
# Avoid padding issue
|
|
||||||
dict(**mm_kwargs, return_tensors=None),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
return mm_data, hf_processor_mm_kwargs
|
||||||
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
# Missing batch dimension
|
# Missing batch dimension
|
||||||
if is_list_of(outputs["input_ids"], int):
|
input_ids = processed_data.get("input_ids")
|
||||||
outputs["input_ids"] = [outputs["input_ids"]]
|
if is_list_of(input_ids, int):
|
||||||
|
processed_data["input_ids"] = [input_ids]
|
||||||
|
|
||||||
return outputs
|
return processed_data
|
||||||
|
|
||||||
def _get_prompt_updates(
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
@@ -287,16 +295,6 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
|||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
def _cached_apply_hf_processor(
|
|
||||||
self,
|
|
||||||
inputs: ProcessorInputs,
|
|
||||||
timing_ctx: TimingContext,
|
|
||||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
|
||||||
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx)
|
|
||||||
|
|
||||||
# NOTE: The tokens are already inserted by the chat template
|
|
||||||
return prompt_ids, mm_info, True
|
|
||||||
|
|
||||||
|
|
||||||
@MULTIMODAL_REGISTRY.register_processor(
|
@MULTIMODAL_REGISTRY.register_processor(
|
||||||
VoxtralMultiModalProcessor,
|
VoxtralMultiModalProcessor,
|
||||||
|
|||||||
@@ -756,27 +756,36 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
|
|||||||
# for encoder profiling.
|
# for encoder profiling.
|
||||||
return [0]
|
return [0]
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
) -> BatchFeature:
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
if mm_data:
|
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
|
||||||
mm_data = dict(audio=mm_data.pop("audios"))
|
mm_data = dict(mm_data)
|
||||||
mm_kwargs = dict(
|
mm_data["audio"] = mm_data.pop("audios")
|
||||||
**mm_kwargs,
|
|
||||||
sampling_rate=feature_extractor.sampling_rate,
|
hf_processor_mm_kwargs = dict(
|
||||||
)
|
**hf_processor_mm_kwargs,
|
||||||
processed_outputs = super()._call_hf_processor(
|
sampling_rate=feature_extractor.sampling_rate,
|
||||||
prompt=prompt,
|
|
||||||
mm_data=mm_data,
|
|
||||||
mm_kwargs=mm_kwargs,
|
|
||||||
)
|
)
|
||||||
if "labels" in processed_outputs:
|
|
||||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
return mm_data, hf_processor_mm_kwargs
|
||||||
return processed_outputs
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
|
) -> BatchFeature:
|
||||||
|
if "labels" in processed_data:
|
||||||
|
processed_data["input_ids"] = processed_data.pop("labels")
|
||||||
|
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -649,13 +649,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
|
|||||||
processor_data["videos"] = raw_videos
|
processor_data["videos"] = raw_videos
|
||||||
return processor_data, passthrough_data
|
return processor_data, passthrough_data
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
self,
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ from vllm.multimodal.inputs import (
|
|||||||
MultiModalFieldConfig,
|
MultiModalFieldConfig,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
from vllm.multimodal.parse import MultiModalDataParser
|
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
|
||||||
from vllm.multimodal.processing import (
|
from vllm.multimodal.processing import (
|
||||||
BaseDummyInputsBuilder,
|
BaseDummyInputsBuilder,
|
||||||
BaseMultiModalProcessor,
|
BaseMultiModalProcessor,
|
||||||
@@ -135,8 +135,8 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
|
|||||||
num_images = mm_counts.get("image", 0)
|
num_images = mm_counts.get("image", 0)
|
||||||
num_audios = mm_counts.get("audio", 0)
|
num_audios = mm_counts.get("audio", 0)
|
||||||
# Use spellings the renderer would emit; tokenization is bypassed in
|
# Use spellings the renderer would emit; tokenization is bypassed in
|
||||||
# _call_hf_processor (we build input_ids directly), so the exact text
|
# _apply_hf_processor_main (we build input_ids directly), so the exact
|
||||||
# only needs to be a stable per-item marker.
|
# text only needs to be a stable per-item marker.
|
||||||
return ("<|content_image|>" * num_images) + (
|
return ("<|content_image|>" * num_images) + (
|
||||||
"<|content_audio_input|>" * num_audios
|
"<|content_audio_input|>" * num_audios
|
||||||
)
|
)
|
||||||
@@ -179,17 +179,19 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]):
|
class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
# Inkling is not a standard HF processor (no fused text+mm call), so we run
|
valid_mm_items = mm_items.select(
|
||||||
# the vendored extractors ourselves and tokenize the text separately.
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
# The MM placeholders in `prompt` are expanded later by the prompt
|
)
|
||||||
# updates, so here we emit ONE placeholder id per media item.
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
|
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
|
|
||||||
images = mm_data.get("images") or []
|
images = mm_data.get("images") or []
|
||||||
@@ -200,7 +202,7 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
|
|||||||
audios = list(cast(Iterable[Any], audios))
|
audios = list(cast(Iterable[Any], audios))
|
||||||
|
|
||||||
prompt_ids = self._tokenize_with_placeholders(
|
prompt_ids = self._tokenize_with_placeholders(
|
||||||
prompt, tokenizer, len(images), len(audios)
|
prompt_text, tokenizer, len(images), len(audios)
|
||||||
)
|
)
|
||||||
|
|
||||||
data: dict[str, Any] = {"input_ids": [prompt_ids]}
|
data: dict[str, Any] = {"input_ids": [prompt_ids]}
|
||||||
@@ -232,7 +234,9 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
|
|||||||
data["input_audio_features"] = input_audio_features
|
data["input_audio_features"] = input_audio_features
|
||||||
data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)
|
data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)
|
||||||
|
|
||||||
return BatchFeature(data=data, tensor_type=None)
|
processed_data = BatchFeature(data=data, tensor_type=None)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _tokenize_with_placeholders(
|
def _tokenize_with_placeholders(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -231,19 +231,6 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
|
|||||||
class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
|
class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
|
||||||
"""Image-only multi-modal processor for Kimi-K3."""
|
"""Image-only multi-modal processor for Kimi-K3."""
|
||||||
|
|
||||||
def _call_hf_processor(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
mm_data: Mapping[str, object],
|
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
# Override so the base always routes through the text+mm path
|
|
||||||
# (`KimiK3Processor.__call__`). Otherwise the mm-only fast path calls
|
|
||||||
# the checkpoint image processor directly with bare PIL images, but it
|
|
||||||
# requires `{"type": "image", "image": PIL}` media dicts that only our
|
|
||||||
# wrapper builds.
|
|
||||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
hf_inputs: BatchFeature,
|
hf_inputs: BatchFeature,
|
||||||
|
|||||||
@@ -306,12 +306,21 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing
|
|||||||
class MiniMaxM3VLMultiModalProcessor(
|
class MiniMaxM3VLMultiModalProcessor(
|
||||||
BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo]
|
BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo]
|
||||||
):
|
):
|
||||||
def _call_hf_processor(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
mm_items: MultiModalDataItems,
|
||||||
mm_data: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
|
if not mm_data:
|
||||||
|
return BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||||
|
|
||||||
mm_data = dict(mm_data)
|
mm_data = dict(mm_data)
|
||||||
# With ``video_needs_metadata=True`` each video arrives as a
|
# With ``video_needs_metadata=True`` each video arrives as a
|
||||||
# ``(frames, metadata)`` tuple. Split the frames back out and forward the
|
# ``(frames, metadata)`` tuple. Split the frames back out and forward the
|
||||||
@@ -340,15 +349,17 @@ class MiniMaxM3VLMultiModalProcessor(
|
|||||||
|
|
||||||
# Override the video processor's default do_resize=False (set for a
|
# Override the video processor's default do_resize=False (set for a
|
||||||
# pre-resized pipeline) to True for vLLM's raw-frame inputs.
|
# pre-resized pipeline) to True for vLLM's raw-frame inputs.
|
||||||
merged = dict(do_resize=True, **mm_kwargs)
|
merged = dict(do_resize=True, **hf_processor_mm_kwargs)
|
||||||
data = dict(text=prompt, **mm_data)
|
data = dict(text=prompt_text, **mm_data)
|
||||||
if video_metadata is not None:
|
if video_metadata is not None:
|
||||||
data["video_metadata"] = video_metadata
|
data["video_metadata"] = video_metadata
|
||||||
return self.info.ctx.call_hf_processor(
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
self.info.get_hf_processor(**mm_kwargs),
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
data,
|
data,
|
||||||
merged,
|
merged,
|
||||||
)
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
return processed_data
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -262,9 +262,6 @@ class InputProcessingContext:
|
|||||||
hf_processor: Callable[..., BatchFeature] | ProcessorMixin,
|
hf_processor: Callable[..., BatchFeature] | ProcessorMixin,
|
||||||
data: Mapping[str, object],
|
data: Mapping[str, object],
|
||||||
kwargs: Mapping[str, object] = {},
|
kwargs: Mapping[str, object] = {},
|
||||||
*,
|
|
||||||
num_tries: int = 1,
|
|
||||||
max_tries: int = 5,
|
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
"""
|
"""
|
||||||
Call `hf_processor` on the prompt `data`
|
Call `hf_processor` on the prompt `data`
|
||||||
@@ -274,6 +271,12 @@ class InputProcessingContext:
|
|||||||
|
|
||||||
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
|
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
|
||||||
|
|
||||||
|
# vLLM needs the full untruncated sequence to keep multi-modal
|
||||||
|
# placeholder tokens aligned; note that the text inputs in
|
||||||
|
# call_hf_processor are just dummy text, not the original prompt.
|
||||||
|
# The original prompt is already tokenized by the renderer.
|
||||||
|
merged_kwargs.setdefault("truncation", False)
|
||||||
|
|
||||||
allowed_kwargs = get_allowed_kwarg_only_overrides(
|
allowed_kwargs = get_allowed_kwarg_only_overrides(
|
||||||
hf_processor,
|
hf_processor,
|
||||||
merged_kwargs,
|
merged_kwargs,
|
||||||
|
|||||||
@@ -91,10 +91,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
|||||||
# prompt tokens.
|
# prompt tokens.
|
||||||
dummy_prompt = []
|
dummy_prompt = []
|
||||||
else:
|
else:
|
||||||
dummy_prompt = tokenizer.encode(
|
dummy_prompt = tokenizer.encode(dummy_text, truncation=False)
|
||||||
dummy_text,
|
|
||||||
**self.info.default_tok_params.get_encode_kwargs(),
|
|
||||||
)
|
|
||||||
|
|
||||||
return ProcessorInputs(
|
return ProcessorInputs(
|
||||||
prompt=dummy_prompt,
|
prompt=dummy_prompt,
|
||||||
|
|||||||
@@ -5,9 +5,10 @@ from collections import defaultdict, deque
|
|||||||
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
|
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
|
||||||
from dataclasses import dataclass, field, replace
|
from dataclasses import dataclass, field, replace
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from functools import lru_cache, partial
|
from functools import lru_cache
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
|
ClassVar,
|
||||||
Generic,
|
Generic,
|
||||||
NamedTuple,
|
NamedTuple,
|
||||||
Protocol,
|
Protocol,
|
||||||
@@ -28,7 +29,6 @@ from vllm.inputs import (
|
|||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.transformers_utils.processor import call_hf_processor_mm_only
|
|
||||||
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
|
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
|
||||||
|
|
||||||
from ..inputs import (
|
from ..inputs import (
|
||||||
@@ -1177,6 +1177,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
Not to be confused with `transformers.ProcessorMixin`.
|
Not to be confused with `transformers.ProcessorMixin`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
hf_processor_applies_updates: ClassVar[bool] = False
|
||||||
|
"""
|
||||||
|
Only set to True if the tokenizer or chat template
|
||||||
|
(which are run before MM processing) inserts placeholder tokens.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
info: _I,
|
info: _I,
|
||||||
@@ -1302,131 +1308,100 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
|
|
||||||
return processor_data, passthrough_data
|
return processor_data, passthrough_data
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None:
|
||||||
|
"""
|
||||||
|
Get the text to pass to the HF processor alongside the multi-modal
|
||||||
|
data.
|
||||||
|
|
||||||
|
By default, no text is passed. If the HF processor requires that
|
||||||
|
text and multi-modal items correspond to each other, you should
|
||||||
|
override this method to return dummy text generated by
|
||||||
|
[`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder].
|
||||||
|
"""
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _preprocess_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
|
||||||
# Not to be confused with `mm_data` in `self.apply`.
|
|
||||||
# This refers to the data to be passed to HF processor.
|
|
||||||
mm_data: Mapping[str, object],
|
mm_data: Mapping[str, object],
|
||||||
mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
|
"""
|
||||||
|
Pre-process the multi-modal data and HF processor keyword arguments
|
||||||
|
before they are passed to the HF processor.
|
||||||
|
|
||||||
|
By default, both are returned as-is. If the HF processor expects the
|
||||||
|
multi-modal data under different keys than those provided by the
|
||||||
|
multi-modal items (e.g. `audio` instead of `audios`), or requires
|
||||||
|
additional keyword arguments (e.g. `sampling_rate`), you should
|
||||||
|
override this method.
|
||||||
|
"""
|
||||||
|
return mm_data, hf_processor_mm_kwargs
|
||||||
|
|
||||||
|
def _postprocess_hf_mm_data(
|
||||||
|
self,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
processed_data: BatchFeature,
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
"""
|
"""
|
||||||
Call the HF processor on the prompt text and
|
Post-process the output of the HF processor.
|
||||||
associated multi-modal data.
|
|
||||||
|
By default, the output is returned as-is. If you need to modify the
|
||||||
|
output of the HF processor before it is converted into multi-modal
|
||||||
|
keyword arguments, you should override this method.
|
||||||
"""
|
"""
|
||||||
return self.info.ctx.call_hf_processor(
|
|
||||||
self.info.get_hf_processor(**mm_kwargs),
|
|
||||||
dict(text=prompt, **mm_data),
|
|
||||||
mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _apply_hf_processor_text_mm(
|
|
||||||
self,
|
|
||||||
prompt_text: str,
|
|
||||||
mm_items: MultiModalDataItems,
|
|
||||||
hf_processor_mm_kwargs: Mapping[str, object],
|
|
||||||
) -> tuple[list[int], BatchFeature]:
|
|
||||||
"""
|
|
||||||
Apply the HF processor on the prompt text and multi-modal data
|
|
||||||
together.
|
|
||||||
"""
|
|
||||||
valid_mm_items = mm_items.select(
|
|
||||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
|
||||||
)
|
|
||||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
|
||||||
|
|
||||||
processed_data = self._call_hf_processor(
|
|
||||||
prompt=prompt_text,
|
|
||||||
mm_data=processor_data,
|
|
||||||
mm_kwargs=hf_processor_mm_kwargs,
|
|
||||||
)
|
|
||||||
processed_data.update(passthrough_data)
|
|
||||||
|
|
||||||
input_ids = processed_data.pop("input_ids")
|
|
||||||
if not isinstance(input_ids, list):
|
|
||||||
input_ids = input_ids.tolist()
|
|
||||||
|
|
||||||
(prompt_ids,) = input_ids
|
|
||||||
|
|
||||||
return prompt_ids, processed_data
|
|
||||||
|
|
||||||
def _apply_hf_processor_tokens_only(
|
|
||||||
self,
|
|
||||||
prompt_tokens: list[int],
|
|
||||||
) -> list[int]:
|
|
||||||
"""
|
|
||||||
Apply the HF processor on the prompt tokens only.
|
|
||||||
|
|
||||||
Most HF processors accept prompt text but not prompt tokens.
|
|
||||||
If the HF processor adds or removes tokens that are not related to
|
|
||||||
multi-modal data, you should override this method so it is consistent
|
|
||||||
with the tokenization of the corresponding text.
|
|
||||||
"""
|
|
||||||
return prompt_tokens
|
|
||||||
|
|
||||||
def _apply_hf_processor_mm_only(
|
|
||||||
self,
|
|
||||||
mm_items: MultiModalDataItems,
|
|
||||||
hf_processor_mm_kwargs: Mapping[str, object],
|
|
||||||
) -> BatchFeature:
|
|
||||||
"""
|
|
||||||
Apply the HF processor on the multi-modal data only.
|
|
||||||
|
|
||||||
Since HF processor requires that text and multi-modal items
|
|
||||||
correspond to each other, we generate dummy text using
|
|
||||||
[`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
|
|
||||||
to go along with the multi-modal data.
|
|
||||||
"""
|
|
||||||
# Custom logic based on text inputs
|
|
||||||
if type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor:
|
|
||||||
mm_counts = mm_items.get_all_counts()
|
|
||||||
|
|
||||||
_, mm_processed_data = self._apply_hf_processor_text_mm(
|
|
||||||
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
|
|
||||||
mm_items=mm_items,
|
|
||||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
return mm_processed_data
|
|
||||||
|
|
||||||
valid_mm_items = mm_items.select(
|
|
||||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
|
||||||
)
|
|
||||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
|
||||||
|
|
||||||
processed_data = self.info.ctx.call_hf_processor(
|
|
||||||
partial(
|
|
||||||
call_hf_processor_mm_only,
|
|
||||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
|
||||||
),
|
|
||||||
processor_data,
|
|
||||||
hf_processor_mm_kwargs,
|
|
||||||
)
|
|
||||||
processed_data.update(passthrough_data)
|
|
||||||
|
|
||||||
return processed_data
|
return processed_data
|
||||||
|
|
||||||
def _apply_hf_processor_main(
|
def _apply_hf_processor_main(
|
||||||
self,
|
self,
|
||||||
prompt: list[int],
|
|
||||||
mm_items: MultiModalDataItems,
|
mm_items: MultiModalDataItems,
|
||||||
hf_processor_mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
) -> tuple[list[int], BatchFeature, bool]:
|
) -> BatchFeature:
|
||||||
"""
|
"""
|
||||||
Apply the HF processor on the prompt tokens and multi-modal data.
|
Apply the HF processor on the multi-modal data.
|
||||||
|
|
||||||
In addition, return whether prompt updates have been applied
|
|
||||||
(`False` here, since for token prompts the updates are normally
|
|
||||||
applied separately via `_maybe_apply_prompt_updates`).
|
|
||||||
"""
|
"""
|
||||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
|
valid_mm_items = mm_items.select(
|
||||||
|
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||||
|
)
|
||||||
|
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||||
|
|
||||||
mm_processed_data = self._apply_hf_processor_mm_only(
|
if processor_data:
|
||||||
mm_items=mm_items,
|
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
|
||||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
processor_data, hf_processor_mm_kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
|
||||||
|
if prompt_text is not None:
|
||||||
|
processor_data = dict(text=prompt_text, **processor_data)
|
||||||
|
|
||||||
|
processed_data = self.info.ctx.call_hf_processor(
|
||||||
|
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||||
|
processor_data,
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
|
)
|
||||||
|
processed_data.update(passthrough_data)
|
||||||
|
else:
|
||||||
|
from transformers.feature_extraction_utils import BatchFeature
|
||||||
|
|
||||||
|
processed_data = BatchFeature(dict(passthrough_data))
|
||||||
|
|
||||||
|
return self._postprocess_hf_mm_data(
|
||||||
|
processor_data,
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
|
processed_data,
|
||||||
)
|
)
|
||||||
|
|
||||||
return prompt_ids, mm_processed_data, False
|
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||||
|
"""
|
||||||
|
Post-process the prompt token IDs before locating or applying
|
||||||
|
multi-modal prompt updates.
|
||||||
|
|
||||||
|
By default, the prompt is returned as-is. If the HF processor (or
|
||||||
|
chat template) applies additional transformations to the prompt
|
||||||
|
that are not reflected in its multi-modal outputs, you should
|
||||||
|
override this method to replicate them.
|
||||||
|
"""
|
||||||
|
return prompt
|
||||||
|
|
||||||
def _get_cache_missing_items(
|
def _get_cache_missing_items(
|
||||||
self,
|
self,
|
||||||
@@ -1531,14 +1506,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
self,
|
self,
|
||||||
inputs: ProcessorInputs,
|
inputs: ProcessorInputs,
|
||||||
timing_ctx: TimingContext,
|
timing_ctx: TimingContext,
|
||||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
) -> MultiModalProcessingInfo:
|
||||||
with timing_ctx.record("apply_hf_processor"):
|
with timing_ctx.record("apply_hf_processor"):
|
||||||
(
|
mm_processed_data = self._apply_hf_processor_main(
|
||||||
prompt_ids,
|
|
||||||
mm_processed_data,
|
|
||||||
is_update_applied,
|
|
||||||
) = self._apply_hf_processor_main(
|
|
||||||
prompt=inputs.prompt,
|
|
||||||
mm_items=inputs.mm_data_items,
|
mm_items=inputs.mm_data_items,
|
||||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
@@ -1569,13 +1539,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
prompt_updates=mm_prompt_updates,
|
prompt_updates=mm_prompt_updates,
|
||||||
)
|
)
|
||||||
|
|
||||||
return prompt_ids, mm_info, is_update_applied
|
return mm_info
|
||||||
|
|
||||||
def _cached_apply_hf_processor(
|
def _cached_apply_hf_processor(
|
||||||
self,
|
self,
|
||||||
inputs: ProcessorInputs,
|
inputs: ProcessorInputs,
|
||||||
timing_ctx: TimingContext,
|
timing_ctx: TimingContext,
|
||||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
) -> MultiModalProcessingInfo:
|
||||||
"""
|
"""
|
||||||
Apply the HF processor on the full prompt text,
|
Apply the HF processor on the full prompt text,
|
||||||
caching the results and reusing cached results.
|
caching the results and reusing cached results.
|
||||||
@@ -1599,16 +1569,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
mm_hashes=mm_hashes,
|
mm_hashes=mm_hashes,
|
||||||
)
|
)
|
||||||
|
|
||||||
# NOTE: `prompt` does not correspond to `mm_missing_data_items`,
|
# NOTE: The prompt does not correspond to `mm_missing_data_items`,
|
||||||
# so we can't apply prompt updates until the new multimodal
|
# so we can't apply prompt updates until the new multimodal
|
||||||
# items are combined with the cached multimodal items
|
# items are combined with the cached multimodal items
|
||||||
with timing_ctx.record("apply_hf_processor"):
|
with timing_ctx.record("apply_hf_processor"):
|
||||||
(
|
mm_missing_processed_data = self._apply_hf_processor_main(
|
||||||
prompt_ids,
|
|
||||||
mm_missing_processed_data,
|
|
||||||
is_update_applied,
|
|
||||||
) = self._apply_hf_processor_main(
|
|
||||||
prompt=inputs.prompt,
|
|
||||||
mm_items=mm_missing_data_items,
|
mm_items=mm_missing_data_items,
|
||||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||||
)
|
)
|
||||||
@@ -1641,7 +1606,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
prompt_updates=mm_prompt_updates,
|
prompt_updates=mm_prompt_updates,
|
||||||
)
|
)
|
||||||
|
|
||||||
return prompt_ids, mm_info, is_update_applied
|
return mm_info
|
||||||
|
|
||||||
def _apply_token_matches(
|
def _apply_token_matches(
|
||||||
self,
|
self,
|
||||||
@@ -1770,7 +1735,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
"There is likely a problem with your "
|
"There is likely a problem with your "
|
||||||
"implementation of merged multi-modal processor for this "
|
"implementation of merged multi-modal processor for this "
|
||||||
"model (usually arising from an inconsistency between "
|
"model (usually arising from an inconsistency between "
|
||||||
"`_call_hf_processor` and `_get_mm_fields_config`)."
|
"`_apply_hf_processor_main` and `_get_mm_fields_config`)."
|
||||||
)
|
)
|
||||||
|
|
||||||
def _validate_mm_updates(
|
def _validate_mm_updates(
|
||||||
@@ -1805,8 +1770,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
f"Expected there to be {item_count} prompt placeholders "
|
f"Expected there to be {item_count} prompt placeholders "
|
||||||
f"corresponding to {item_count} {modality} items, but "
|
f"corresponding to {item_count} {modality} items, but "
|
||||||
f"instead found {len(placeholders)} prompt placeholders! "
|
f"instead found {len(placeholders)} prompt placeholders! "
|
||||||
"Make sure the implementation of `_call_hf_processor` and "
|
"Make sure the implementation of `_apply_hf_processor_main` "
|
||||||
"`_get_mm_fields_config` are consistent with each other."
|
"and `_get_mm_fields_config` are consistent with each other."
|
||||||
)
|
)
|
||||||
|
|
||||||
def _maybe_apply_prompt_updates(
|
def _maybe_apply_prompt_updates(
|
||||||
@@ -1854,11 +1819,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
3. Extract information about the placeholder tokens from the
|
3. Extract information about the placeholder tokens from the
|
||||||
processed token IDs.
|
processed token IDs.
|
||||||
"""
|
"""
|
||||||
(
|
prompt_ids = self._postprocess_prompt(inputs.prompt)
|
||||||
prompt_ids,
|
mm_info = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||||
mm_info,
|
|
||||||
is_update_applied,
|
|
||||||
) = self._cached_apply_hf_processor(inputs, timing_ctx)
|
|
||||||
|
|
||||||
with timing_ctx.record("apply_prompt_updates"):
|
with timing_ctx.record("apply_prompt_updates"):
|
||||||
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
|
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
|
||||||
@@ -1866,7 +1828,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
prompt_ids=prompt_ids,
|
prompt_ids=prompt_ids,
|
||||||
mm_kwargs=mm_info.kwargs,
|
mm_kwargs=mm_info.kwargs,
|
||||||
mm_prompt_updates=mm_info.prompt_updates,
|
mm_prompt_updates=mm_info.prompt_updates,
|
||||||
is_update_applied=is_update_applied,
|
is_update_applied=self.hf_processor_applies_updates,
|
||||||
)
|
)
|
||||||
|
|
||||||
mm_placeholder_ranges = {
|
mm_placeholder_ranges = {
|
||||||
|
|||||||
@@ -11,17 +11,13 @@ from transformers import (
|
|||||||
AutoImageProcessor,
|
AutoImageProcessor,
|
||||||
AutoProcessor,
|
AutoProcessor,
|
||||||
AutoVideoProcessor,
|
AutoVideoProcessor,
|
||||||
BatchFeature,
|
|
||||||
processing_utils,
|
processing_utils,
|
||||||
)
|
)
|
||||||
from transformers.audio_utils import AudioInput
|
|
||||||
from transformers.feature_extraction_utils import FeatureExtractionMixin
|
from transformers.feature_extraction_utils import FeatureExtractionMixin
|
||||||
from transformers.image_processing_utils import BaseImageProcessor
|
from transformers.image_processing_utils import BaseImageProcessor
|
||||||
from transformers.image_utils import ImageInput
|
|
||||||
from transformers.models.auto.video_processing_auto import VIDEO_PROCESSOR_MAPPING_NAMES
|
from transformers.models.auto.video_processing_auto import VIDEO_PROCESSOR_MAPPING_NAMES
|
||||||
from transformers.processing_utils import ProcessorMixin
|
from transformers.processing_utils import ProcessorMixin
|
||||||
from transformers.video_processing_utils import BaseVideoProcessor
|
from transformers.video_processing_utils import BaseVideoProcessor
|
||||||
from transformers.video_utils import VideoInput
|
|
||||||
from typing_extensions import TypeVar
|
from typing_extensions import TypeVar
|
||||||
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
@@ -554,43 +550,3 @@ def cached_video_processor_from_config(
|
|||||||
processor_cls_overrides=processor_cls, # type: ignore[arg-type]
|
processor_cls_overrides=processor_cls, # type: ignore[arg-type]
|
||||||
**_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs),
|
**_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def call_hf_processor_mm_only(
|
|
||||||
processor: ProcessorMixin,
|
|
||||||
images: ImageInput | None = None,
|
|
||||||
videos: VideoInput | None = None,
|
|
||||||
audio: AudioInput | None = None,
|
|
||||||
**kwargs,
|
|
||||||
) -> BatchFeature:
|
|
||||||
output_kwargs = processor._merge_kwargs(
|
|
||||||
get_processor_kwargs_type(processor),
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
if audio is not None and (
|
|
||||||
feature_extractor := getattr(processor, "feature_extractor", None)
|
|
||||||
):
|
|
||||||
audio_inputs = feature_extractor(audio, **output_kwargs["audio_kwargs"])
|
|
||||||
audio_inputs["feature_attention_mask"] = audio_inputs.pop("attention_mask")
|
|
||||||
else:
|
|
||||||
audio_inputs = {}
|
|
||||||
|
|
||||||
if images is not None and (
|
|
||||||
image_processor := getattr(processor, "image_processor", None)
|
|
||||||
):
|
|
||||||
images_inputs = image_processor(images=images, **output_kwargs["images_kwargs"])
|
|
||||||
else:
|
|
||||||
images_inputs = {}
|
|
||||||
|
|
||||||
if videos is not None and (
|
|
||||||
video_processor := getattr(processor, "video_processor", None)
|
|
||||||
):
|
|
||||||
videos_inputs = video_processor(videos=videos, **output_kwargs["videos_kwargs"])
|
|
||||||
else:
|
|
||||||
videos_inputs = {}
|
|
||||||
|
|
||||||
return BatchFeature(
|
|
||||||
data={**audio_inputs, **images_inputs, **videos_inputs},
|
|
||||||
tensor_type=kwargs.get("return_tensors"),
|
|
||||||
)
|
|
||||||
|
|||||||
Reference in New Issue
Block a user