[MM] Further cleanup _apply_hf_processor_main (#53610)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-16 10:21:00 -07:00
committed by GitHub
parent 6ca2b23e22
commit 0b9e018fb9
74 changed files with 1048 additions and 1323 deletions
+22 -11
View File
@@ -454,17 +454,17 @@ return a schema of the tensors outputted by the HF processor that are related to
??? code
```python
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
# Mistral3Processor requires text corresponding to the images
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
if not hf_data:
return processed_data
pixel_values = processed_data.get("pixel_values")
@@ -486,25 +486,36 @@ return a schema of the tensors outputted by the HF processor that are related to
calls the HF processor on the multi-modal data without passing any text.
If the HF processor instead requires text corresponding to the multi-modal items,
you should override
[_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text]
[_get_hf_mm_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_text]
to return the dummy text from
[BaseDummyInputsBuilder.get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text]
like in the example above. If the HF processor expects the multi-modal data
under different keys than those provided by the multi-modal items
(e.g. `audio` instead of `audios`), or requires additional keyword arguments
under different keys than those provided by the multi-modal items,
or requires additional keyword arguments
(e.g. `sampling_rate`), you should override
[_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data].
[_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].
If you need to modify the output of the HF processor,
you should override
[_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data].
Passthrough fields have already been merged into `processed_data` when this
hook is called.
If you need to change how the normalized inputs are passed to the HF
processor (for example, to call a method other than `__call__`, or to use
different kwargs to construct and invoke it), override
[_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor].
If the HF processor applies additional transformations to the prompt tokens
that are not reflected in its multi-modal outputs (e.g. prepending a BOS token),
you should override
[_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt]
to replicate them.
For even more control over how the HF processor is called, you can override
Only processors that require multiple HF calls or manually construct the
complete output should override
[_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]
directly.
directly. Such implementations must obtain their normalized inputs through
[_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs]
and return through
[_finalize_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._finalize_hf_mm_data]
so passthrough fields and model-specific postprocessing are applied consistently.
Since `pixel_values` is now a list with one tensor per image, we can override
[_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows:
@@ -724,7 +735,7 @@ Examples:
### Custom HF processor
Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. It is then applied to the multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor] inside [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main].
Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. The default [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor] applies it to the normalized multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor]. Override `_call_hf_processor` if the custom processor instead exposes a different callable interface.
Examples:
+9 -2
View File
@@ -8,9 +8,16 @@ In vLLM's rendering pipeline (see [BaseRenderer][vllm.renderers.base.BaseRendere
Since Transformers 5.10, `ProcessorMixin` now allows multi-modal inputs to be passed by themselves. However, certain subclasses (such as `ChameleonProcessor`) and older out-of-tree implementations may still define their own `__call__` method that assumes the presence of text with corresponding placeholder tokens. This causes a problem as we don't have the original text anymore to pass to these processors.
To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text] returns so that [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] passes it to the HF processor together with the multi-modal inputs to obtain the processed multi-modal data.
To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_mm_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_text] returns. [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs] adds this text and `truncation=False` to the normalized HF inputs. The truncation setting is attached only to calls that use dummy text, so unrelated processor calls do not receive a text-specific option.
Similarly, since the multi-modal data extracted by vLLM may not match what a specific HF processor expects, [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] allows each model to adapt the inputs via [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data] (e.g. renaming keys like `audios` to `audio` or injecting extra keyword arguments such as `sampling_rate`) and the outputs via [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data], without having to reimplement the entire method.
HF processing is split into four overridable stages:
1. [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs] extracts and normalizes processor data, kwargs, and passthrough data. It centrally renames vLLM's `audios` field to the standard HF `audio` keyword; processors that require another spelling can adapt it in an override.
2. [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor] invokes the HF processor. Models can override it when they use a different callable or need separate construction and invocation kwargs.
3. [_finalize_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._finalize_hf_mm_data] merges passthrough fields into the processed output. It lazily creates an empty output when there is no processor data.
4. [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data] performs model-specific output transformations after passthrough fields have been merged.
This division lets most models customize one stage without reimplementing [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]. Implementations that genuinely require multiple processor calls or manually construct their complete output may still override that method, but must use `_get_hf_mm_inputs` and `_finalize_hf_mm_data` to retain the common input and output behavior.
## Prompt Update Detection
+1 -13
View File
@@ -374,21 +374,9 @@ class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):
class AudioFlamingo3MultiModalProcessor(
BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", None)
if audios is not None:
mm_data["audio"] = audios
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
@@ -119,7 +119,7 @@ class BailingMoeV3VLDummyInputsBuilder(
class BailingMoeV3VLMultiModalProcessor(
BaseMultiModalProcessor[BailingMoeV3VLProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_prompt_updates(
+1 -1
View File
@@ -222,7 +222,7 @@ class Cohere2VisionDummyInputsBuilder(
class Cohere2VisionMultiModalProcessor(
BaseMultiModalProcessor[Cohere2VisionProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+18 -28
View File
@@ -8,7 +8,7 @@ from typing import Any, ClassVar
import torch
import torch.nn.functional as F
from torch import nn
from transformers import BatchFeature, PretrainedConfig
from transformers import PretrainedConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
@@ -50,7 +50,10 @@ from vllm.multimodal.processing import (
PromptUpdate,
TimingContext,
)
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
from vllm.multimodal.processing.processor import (
HFMultiModalInputs,
MultiModalProcessingResult,
)
from vllm.renderers import TokenizeParams
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.transformers_utils.processors.cohere_asr import (
@@ -1962,42 +1965,29 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
) -> list[int]:
return [0]
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
return hf_inputs._replace(
hf_kwargs=dict(
hf_inputs.hf_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
# Dithering injects noise into the extracted features, so the
# feature extractor is not a pure function of its input. Since the
# processing cache assumes that processor outputs are invariant
+7 -7
View File
@@ -157,15 +157,14 @@ class ColModernVBertMultiModalProcessor(
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
@@ -193,8 +192,9 @@ class ColModernVBertMultiModalProcessor(
result.update(image_outputs)
processed_data = result
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
mm_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+7 -18
View File
@@ -20,7 +20,7 @@ from collections.abc import Iterable, Mapping
import torch
import torch.nn as nn
from transformers import BatchFeature, PaliGemmaProcessor
from transformers import PaliGemmaProcessor
from vllm.config import VllmConfig
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing.processor import HFMultiModalInputs
from .interfaces import SupportsLateInteraction
from .interfaces_base import default_pooling_type
@@ -59,31 +60,19 @@ class ColPaliProcessingInfo(PaliGemmaProcessingInfo):
class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
"""Multimodal processor for ColPali."""
def _apply_hf_processor_main(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# The ColPali tokenizer_config.json ships with a small default
# max_length (50) that truncates the 1024 image tokens inserted
# by PaliGemmaProcessor, causing a token-count mismatch.
# vLLM enforces its own max_model_len, so we disable HF
# truncation to keep all image + text tokens intact.
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
dict(**hf_processor_mm_kwargs, truncation=False),
)
processed_data.update(passthrough_data)
return processed_data
return hf_inputs._replace(hf_kwargs=dict(hf_inputs.hf_kwargs, truncation=False))
@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
+11 -26
View File
@@ -46,10 +46,10 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sampling_params import SamplingParams
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.hf import HfTokenizer
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
from vllm.transformers_utils.processors.deepseek_ocr import (
BASE_SIZE,
@@ -287,34 +287,19 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing
class DeepseekOCRMultiModalProcessor(
BaseMultiModalProcessor[DeepseekOCRProcessingInfo]
):
def _apply_hf_processor_main(
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
if "text" in hf_inputs.hf_data:
hf_inputs.hf_data["prompt"] = hf_inputs.hf_data.pop("text")
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
if mm_data:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(prompt=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
else:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
processed_data = tokenizer(
prompt_text, add_special_tokens=True, return_tensors="pt"
)
processed_data.update(passthrough_data)
return processed_data
return hf_inputs
def _get_mm_fields_config(
self,
+11 -27
View File
@@ -45,9 +45,9 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.hf import HfTokenizer
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
from vllm.transformers_utils.processors.deepseek_ocr import (
BASE_SIZE,
@@ -158,35 +158,19 @@ class DeepseekOCR2DummyInputsBuilder(
class DeepseekOCR2MultiModalProcessor(
BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo]
):
def _apply_hf_processor_main(
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
if "text" in hf_inputs.hf_data:
hf_inputs.hf_data["prompt"] = hf_inputs.hf_data.pop("text")
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
if mm_data:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(prompt=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
else:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
processed_data = tokenizer(
prompt_text, add_special_tokens=True, return_tensors="pt"
)
processed_data.update(passthrough_data)
return processed_data
return hf_inputs
def _get_mm_fields_config(
self,
+3 -3
View File
@@ -35,7 +35,7 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
MultiModalProcessingInfo,
MultiModalProcessingResult,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
@@ -233,7 +233,7 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
class DeepseekVL2MultiModalProcessor(
BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
@@ -307,7 +307,7 @@ class DeepseekVL2MultiModalProcessor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
# The processor logic is different for len(images) <= 2 vs > 2
# Since the processing cache assumes that the processor output is
# invariant of how many images are passed per prompt, we only
+50 -57
View File
@@ -71,6 +71,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
cached_encode,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -1112,34 +1113,32 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
pixel_values = pixel_values.to(hf_config.dtype)
return pixel_values
def _apply_hf_processor_main(
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
hf_data = hf_inputs.hf_data
if "images" not in mm_data and "videos" not in mm_data:
return BatchFeature(dict(passthrough_data))
if "images" not in hf_data and "videos" not in hf_data:
return hf_inputs
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data)
if "images" not in mm_data:
mm_data["images"] = []
if "videos" not in mm_data:
mm_data["videos"] = []
prompt_text = hf_data["text"]
hf_data["text"] = [prompt_text]
hf_data.setdefault("images", [])
hf_data.setdefault("videos", [])
# Check if HF processor supports video metadata
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_inputs.hf_kwargs)
supports_video_metadata = getattr(
hf_processor, "supports_video_metadata", False
)
videos = mm_data["videos"]
videos = hf_data["videos"]
assert isinstance(videos, Sequence)
if videos and not supports_video_metadata:
# Old HF processor, unwrap tuple to pure frames
@@ -1147,49 +1146,43 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
"HF processor doesn't support video metadata. "
"Timestamps will NOT be rendered. Please upgrade the model."
)
mm_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos]
hf_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos]
processor_output = self.info.ctx.call_hf_processor(
hf_processor,
dict(
text=[prompt_text], images=mm_data["images"], videos=mm_data["videos"]
),
hf_processor_mm_kwargs,
)
return hf_inputs
def _postprocess_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "images" not in hf_data and "videos" not in hf_data:
return processed_data
# Divide the processor_output into two modalities: image and video.
if processor_output is not None:
pixel_values = processor_output["images"]
if pixel_values is not None:
processor_output["images"] = self._pixel_values_norm(
pixel_values, hf_processor_mm_kwargs
)
for key in list(processor_output.keys()):
if processor_output[key] is None:
del processor_output[key]
continue
if key == "grid_thw":
grid_thw = processor_output["grid_thw"]
pixel_values_all = processor_output["images"]
# Identify elements where the first
# dimension is greater than 1 and
# treat them as the video modality
mask = grid_thw[:, 0] > 1
processor_output["video_grid_thw"] = grid_thw[mask]
processor_output["image_grid_thw"] = grid_thw[~mask]
image_patch_num = (
processor_output["image_grid_thw"].prod(dim=1).sum()
)
processor_output["pixel_values"] = pixel_values_all[
:image_patch_num
]
processor_output["pixel_values_videos"] = pixel_values_all[
image_patch_num:
]
del processor_output["images"]
pixel_values = processed_data["images"]
if pixel_values is not None:
processed_data["images"] = self._pixel_values_norm(pixel_values, hf_kwargs)
for key in list(processed_data.keys()):
if processed_data[key] is None:
del processed_data[key]
continue
if key == "grid_thw":
grid_thw = processed_data["grid_thw"]
pixel_values_all = processed_data["images"]
# Identify elements where the first
# dimension is greater than 1 and
# treat them as the video modality
mask = grid_thw[:, 0] > 1
processed_data["video_grid_thw"] = grid_thw[mask]
processed_data["image_grid_thw"] = grid_thw[~mask]
image_patch_num = processed_data["image_grid_thw"].prod(dim=1).sum()
processed_data["pixel_values"] = pixel_values_all[:image_patch_num]
processed_data["pixel_values_videos"] = pixel_values_all[
image_patch_num:
]
del processed_data["images"]
processed_data = processor_output
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates(
+13 -25
View File
@@ -38,6 +38,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.processors.fireredasr2 import (
FireRedASR2FeatureExtractor,
@@ -228,37 +229,24 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing
class FireRedASR2MultiModalProcessor(
BaseMultiModalProcessor[FireRedASR2ProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
return hf_inputs._replace(
hf_kwargs=dict(
hf_inputs.hf_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+17 -27
View File
@@ -49,7 +49,10 @@ from vllm.multimodal.processing import (
PromptUpdate,
TimingContext,
)
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
from vllm.multimodal.processing.processor import (
HFMultiModalInputs,
MultiModalProcessingResult,
)
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -752,42 +755,29 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
return hf_inputs._replace(
hf_kwargs=dict(
hf_inputs.hf_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
# Dithering injects noise into the extracted features, so the
# feature extractor is not a pure function of its input. Since the
# processing cache assumes that processor outputs are invariant
+12 -11
View File
@@ -616,29 +616,29 @@ class FunAudioChatMultiModalProcessor(
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
tokenizer = self.info.get_tokenizer()
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
audios = mm_data.get("audios", [])
audios = mm_data.get("audio", [])
if not audios:
processed_data = BatchFeature({"input_ids": input_ids})
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
mm_data, hf_kwargs, passthrough_data, processed_data
)
assert isinstance(audios, Sequence)
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
sr = int(feature_extractor.sampling_rate)
min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400)
@@ -690,8 +690,9 @@ class FunAudioChatMultiModalProcessor(
}
processed_data = BatchFeature({"input_ids": input_ids, **mm_inputs})
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
mm_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+1 -1
View File
@@ -263,7 +263,7 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+1 -12
View File
@@ -265,20 +265,9 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
if "audios" in mm_data:
mm_data["audio"] = mm_data.pop("audios")
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
+20 -23
View File
@@ -595,22 +595,25 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_kwargs)
val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
if val is not None and val not in _SUPPORTED_SOFT_TOKENS:
@@ -619,14 +622,12 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
f"Valid values are {_SUPPORTED_SOFT_TOKENS}."
)
mm_data = dict(mm_data)
# ---- VIDEO HANDLING ----
# Gemma4 decomposes video into timestamped image frames.
# Each frame is processed with max_soft_tokens=70 through the
# same vision tower, matching transformers processing_gemma4.py.
video_outputs: dict[str, Any] = {}
if videos := mm_data.pop("videos", []):
if videos := hf_data.pop("videos", []):
assert isinstance(videos, list)
processor = self.info.get_hf_processor()
@@ -656,7 +657,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
timestamps = [idx / fps for idx in frame_indices]
# Process frames as images with max_soft_tokens=70
video_mm_kwargs = dict(hf_processor_mm_kwargs)
video_mm_kwargs = dict(hf_kwargs)
# Override configured image options too: these inputs are video
# frames, whose cache keys exclude images_kwargs.
video_mm_kwargs["images_kwargs"] = {}
@@ -726,18 +727,14 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
"video_timestamps": video_timestamps_per_video,
}
# The processor accepts 'audio' not 'audios'.
if "audios" in mm_data:
mm_data["audio"] = mm_data.pop("audios")
# Warn if any audio waveform exceeds the model's max duration.
if "audio" in mm_data:
if "audio" in hf_data:
processor = self.info.get_hf_processor()
sr = processor.feature_extractor.sampling_rate
max_tokens = processor.audio_seq_length
ms_per_tok = processor.audio_ms_per_token
max_duration_s = max_tokens * ms_per_tok / 1000.0
audios = mm_data["audio"]
audios = hf_data["audio"]
if not isinstance(audios, (list, tuple)):
audios = [audios]
for i, waveform in enumerate(audios):
@@ -760,13 +757,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
# NOTE: This requires a corresponding type annotation on the
# HF side (Gemma4ProcessorKwargs.images_kwargs) so that
# _merge_kwargs routes max_soft_tokens into images_kwargs.
patched_mm_kwargs = dict(hf_processor_mm_kwargs)
patched_mm_kwargs = dict(hf_kwargs)
if val is not None and is_top_level_max_soft_tokens:
patched_mm_kwargs["max_soft_tokens"] = val
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**patched_mm_kwargs),
dict(text=prompt_text, **mm_data),
dict(text=prompt_text, **hf_data),
patched_mm_kwargs,
)
@@ -797,9 +794,9 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
processed_data["input_features_mask"] = unpadded_masks
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+30 -23
View File
@@ -1582,52 +1582,59 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
return prepared_data, prepared_kwargs
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
mm_data = dict(mm_data)
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
processor = self.info.get_hf_processor(**hf_kwargs)
use_direct_path = (
not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
)
if use_direct_path:
prepared_data, prepared_kwargs = self._get_direct_path_inputs(
mm_data, hf_processor_mm_kwargs
hf_data, hf_kwargs
)
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**prepared_kwargs),
dict(text=prompt_text, **prepared_data),
prepared_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
videos = mm_data.get("videos")
if isinstance(videos, list) and videos:
if (
"videos" in hf_data
and isinstance(videos := hf_data["videos"], list)
and len(videos) > 0
):
# Videos are processed item-wise below; remove them from hf_data so
# the combined call at the end does not re-sample the raw frames.
del hf_data["videos"]
video_grid_thw_lst = []
pixel_values_videos_lst = []
frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor)
swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id
mm_data.pop("videos")
for item in videos:
video_array, metadata = item
# don't update mm_kwargs inplace
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
video_mm_kwargs = dict(**hf_kwargs)
video_mm_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", True
)
@@ -1667,9 +1674,9 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
swap_video_frame_tokens = False
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
self.info.get_hf_processor(**hf_kwargs),
dict(text=prompt_text, **hf_data),
hf_kwargs,
)
if swap_video_frame_tokens:
input_ids = processed_data["input_ids"]
@@ -1677,9 +1684,9 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
processed_data["input_ids"] = input_ids
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+13 -14
View File
@@ -44,6 +44,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.transformers_utils.processor import cached_processor_from_config
@@ -739,26 +740,24 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
chunk_counts.append(min(n_chunks, max_windows))
return chunk_counts
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
if "audios" in mm_data:
mm_data["audio"] = mm_data.pop("audios")
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
return hf_inputs._replace(
hf_kwargs=dict(
hf_inputs.hf_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
+5 -19
View File
@@ -110,7 +110,7 @@ class GraniteSpeechAudioInputs(TensorSchema):
"""List of audio embedding sizes for each item in batch."""
class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
class GraniteSpeechProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
feature_extractor = self.get_hf_processor().audio_processor
@@ -135,7 +135,7 @@ class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
### Input Processing & Multimodal utils
class GraniteSpeechMultiModalProcessor(
BaseMultiModalProcessor[GraniteSpeechMultiModalProcessingInfo]
BaseMultiModalProcessor[GraniteSpeechProcessingInfo]
):
def _get_mm_fields_config(
self,
@@ -181,23 +181,9 @@ class GraniteSpeechMultiModalProcessor(
)
]
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
# GraniteSpeechFeatureExtractor accepts "audio"
mm_data["audio"] = audios
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
@@ -216,7 +202,7 @@ class GraniteSpeechMultiModalProcessor(
class GraniteSpeechDummyInputsBuilder(
BaseDummyInputsBuilder[GraniteSpeechMultiModalProcessingInfo]
BaseDummyInputsBuilder[GraniteSpeechProcessingInfo]
):
def get_dummy_mm_data(
self,
@@ -565,7 +551,7 @@ class GraniteSpeechCTCEncoder(nn.Module):
@MULTIMODAL_REGISTRY.register_processor(
GraniteSpeechMultiModalProcessor,
info=GraniteSpeechMultiModalProcessingInfo,
info=GraniteSpeechProcessingInfo,
dummy_inputs=GraniteSpeechDummyInputsBuilder,
)
class GraniteSpeechForConditionalGeneration(
@@ -12,8 +12,8 @@ from .granite_speech import (
GraniteSpeechCTCEncoder,
GraniteSpeechDummyInputsBuilder,
GraniteSpeechForConditionalGeneration,
GraniteSpeechMultiModalProcessingInfo,
GraniteSpeechMultiModalProcessor,
GraniteSpeechProcessingInfo,
)
ISO639_1_SUPPORTED_LANGS = {
@@ -60,7 +60,7 @@ class GraniteSpeechPlusCTCEncoder(GraniteSpeechCTCEncoder):
@MULTIMODAL_REGISTRY.register_processor(
GraniteSpeechMultiModalProcessor,
info=GraniteSpeechMultiModalProcessingInfo,
info=GraniteSpeechProcessingInfo,
dummy_inputs=GraniteSpeechDummyInputsBuilder,
)
class GraniteSpeechPlusForConditionalGeneration(GraniteSpeechForConditionalGeneration):
+2 -2
View File
@@ -21,7 +21,7 @@ from vllm.multimodal.parse import (
MultiModalDataItems,
)
from vllm.multimodal.processing.processor import (
MultiModalProcessingInfo,
MultiModalProcessingResult,
ProcessorInputs,
PromptReplacement,
TimingContext,
@@ -139,7 +139,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
# The processor logic is different for len(images) <= 1 vs > 1
# Since the processing cache assumes that the processor output is
# invariant of how many images are passed per prompt, we only
@@ -261,44 +261,8 @@ class HCXVisionV2MultiModalProcessor(
):
"""Multimodal processor for HyperCLOVAX V2 (32B Think model)."""
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
images = mm_data.get("images")
videos = mm_data.get("videos")
# Get the HF processor
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
# Build data dict for HF processor (images/videos only)
# NOTE: We pass the prompt as-is without token normalization.
# Token expansion is handled by vLLM via _get_prompt_updates.
data: dict[str, object] = dict(
text=prompt_text,
images=images,
videos=videos,
)
processed_data = self.info.ctx.call_hf_processor(
hf_processor=hf_processor,
data=data,
kwargs=hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_prompt_updates(
self,
+21 -21
View File
@@ -52,6 +52,7 @@ from vllm.multimodal.processing import (
PromptUpdateDetails,
cached_encode,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -240,31 +241,32 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]):
def _apply_hf_processor_main(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
image_processor = self.info.get_hf_processor().image_processor
if getattr(image_processor, "backend", "pil") == "pil":
hf_processor_mm_kwargs = {
"input_data_format": "channels_last",
**hf_processor_mm_kwargs,
}
hf_inputs = hf_inputs._replace(
hf_kwargs={
"input_data_format": "channels_last",
**hf_inputs.hf_kwargs,
}
)
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
hf_processor_mm_kwargs,
)
return hf_inputs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
images = mm_data.get("images", [])
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
@@ -289,8 +291,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
processed_data["pixel_values"].squeeze_(0)
processed_data["pixel_attention_mask"].squeeze_(0)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
+20 -18
View File
@@ -335,28 +335,30 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]):
"""Basic image-only MultiModalProcessor for InternS1-style models."""
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
mm_data = dict(mm_data)
videos = mm_data.pop("videos", [])
images = mm_data.pop("images", [])
videos = hf_data.pop("videos", [])
images = hf_data.pop("images", [])
assert isinstance(videos, list)
assert isinstance(images, list)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_kwargs)
tokenizer = hf_processor.tokenizer
vocab = tokenizer.get_vocab()
@@ -374,9 +376,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
image_pixel_values = []
for image in images:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
self.info.get_hf_processor(**hf_kwargs),
dict(text=hf_processor.image_token, **{"images": image}),
hf_processor_mm_kwargs,
hf_kwargs,
)
image_pixel_values.append(processed_data.pop("pixel_values"))
@@ -398,9 +400,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
video_pixel_values = []
for video in videos:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
self.info.get_hf_processor(**hf_kwargs),
dict(text=hf_processor.video_token, **{"videos": video}),
hf_processor_mm_kwargs,
hf_kwargs,
)
video_pixel_values.append(processed_data.pop("pixel_values"))
@@ -430,9 +432,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
processed_data.update(text_outputs)
processed_data.update(image_outputs)
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+8 -7
View File
@@ -13,7 +13,7 @@ from vllm.model_executor.layers.pooler import DispatchPooler
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalKwargsItems
from vllm.multimodal.processing.processor import (
MultiModalProcessingInfo,
MultiModalProcessingResult,
ProcessorInputs,
TimingContext,
)
@@ -63,19 +63,19 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
mm_info = super()._cached_apply_hf_processor(inputs, timing_ctx)
) -> MultiModalProcessingResult:
mm_res = super()._cached_apply_hf_processor(inputs, timing_ctx)
# Score inputs are query-first, while the prompt template is document-first.
mm_kwargs = MultiModalKwargsItems(
{
modality: list(reversed(items))
for modality, items in mm_info.kwargs.items()
for modality, items in mm_res.kwargs.items()
}
)
mm_hashes = {
modality: list(reversed(hashes))
for modality, hashes in mm_info.hashes.items()
for modality, hashes in mm_res.hashes.items()
}
mm_prompt_updates = {
modality: [
@@ -85,10 +85,11 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
]
for item_idx, updates in enumerate(reversed(item_updates))
]
for modality, item_updates in mm_info.prompt_updates.items()
for modality, item_updates in mm_res.prompt_updates.items()
}
return MultiModalProcessingInfo(
return MultiModalProcessingResult(
prompt_ids=mm_res.prompt_ids,
kwargs=mm_kwargs,
hashes=mm_hashes,
prompt_updates=mm_prompt_updates,
+14 -10
View File
@@ -459,24 +459,27 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
def media_token_id(self) -> int:
return self.info.get_hf_config().text_config.eos_token_id + 1
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Run the underlying HF processor on text and image data."""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data or not mm_data.get("images", []):
return BatchFeature(dict(passthrough_data))
if not hf_data or not hf_data.get("images", []):
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
# Images
image_inputs = mm_data.get("images", [])
image_inputs = hf_data.get("images", [])
pixel_sizes = []
if not isinstance(image_inputs[0], Image.Image):
image_inputs = [Image.fromarray(image) for image in image_inputs]
@@ -530,8 +533,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
pixel_sizes=torch.tensor(pixel_sizes),
)
processed_data = BatchFeature(combined_outputs, tensor_type="pt")
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_prompt_updates(
self,
+1 -1
View File
@@ -1160,7 +1160,7 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):
class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_prompt_updates(
+1 -1
View File
@@ -373,7 +373,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_prompt_updates(
+8 -24
View File
@@ -47,6 +47,7 @@ from vllm.multimodal.processing import (
)
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
HFMultiModalInputs,
ProcessorInputs,
)
from vllm.sequence import IntermediateTensors
@@ -227,27 +228,17 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser):
class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]):
"""vLLM multi-modal processor wrapper for Kimi-Audio."""
def _apply_hf_processor_main(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Call the HuggingFace processor."""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
# Convert mm_data format: {'audios': [...]} -> {'audio': ...}
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# Convert audio format: [(array, sr), ...] -> [array, ...]
# KimiAudioProcessor expects raw numpy arrays
if audios:
mm_data = hf_inputs.hf_data
if audios := mm_data.pop("audio", []):
audio_arrays = []
for aud in audios:
if isinstance(aud, (tuple, list)) and len(aud) == 2:
@@ -259,14 +250,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
audio_arrays.append(aud)
mm_data["audio"] = audio_arrays
# Use the context's call_hf_processor for proper handling
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
return hf_inputs
def _get_mm_fields_config(
self,
+1 -1
View File
@@ -236,7 +236,7 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_mm_fields_config(
+1 -1
View File
@@ -406,7 +406,7 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):
class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+2 -26
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, Sequence
from typing import TypeVar
import torch
import torch.nn as nn
@@ -35,8 +34,6 @@ from vllm.multimodal.processing import (
PromptUpdateDetails,
)
_I = TypeVar("_I", bound=Mistral3ProcessingInfo)
class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
def get_vision_encoder_info(
@@ -49,8 +46,8 @@ class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
return PixtralHFEncoderInfo(self.get_hf_config())
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
@@ -62,27 +59,6 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
if not mm_data:
return processed_data
input_ids = processed_data.get("input_ids")
if input_ids is not None:
processor = self.info.get_hf_processor()
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
break_id = vocab.get(processor.image_break_token)
end_id = vocab.get(processor.image_end_token)
# create mask to remove break/end tokens
keep_mask = ~torch.isin(
input_ids,
torch.tensor([break_id, end_id]),
)
processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0)
if "attention_mask" in processed_data:
processed_data["attention_mask"] = processed_data["attention_mask"][
keep_mask
].unsqueeze(0)
# un-pad pixel_values per-image so caches remain independent.
pixel_values = processed_data.get("pixel_values")
if pixel_values is not None:
+1 -1
View File
@@ -320,7 +320,7 @@ class PixtralHFProcessingInfo(BaseLlavaProcessingInfo):
class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+1 -1
View File
@@ -209,7 +209,7 @@ class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]):
class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_mm_fields_config(
+25 -21
View File
@@ -324,33 +324,36 @@ class LlavaOnevisionMultiModalProcessor(
pixel_values_videos=MultiModalFieldConfig.batched("video"),
)
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
mm_data = dict(mm_data)
videos = mm_data.pop("videos", [])
videos = hf_data.pop("videos", [])
assert isinstance(videos, list)
if not videos:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
self.info.get_hf_processor(**hf_kwargs),
dict(text=prompt_text, **hf_data),
hf_kwargs,
)
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
processed_data.update(passthrough_data)
return processed_data
# LLaVA-OneVision processor doesn't support multiple videos
# with different sizes when converting back to tensors
@@ -360,13 +363,13 @@ class LlavaOnevisionMultiModalProcessor(
image_token = processor.image_token
video_token = processor.video_token
images = mm_data.pop("images", [])
images = hf_data.pop("images", [])
assert isinstance(images, list)
if images:
processor_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
self.info.get_hf_processor(**hf_kwargs),
dict(text=image_token * len(images), **{"images": images}),
hf_processor_mm_kwargs,
hf_kwargs,
)
image_outputs = {
k: v
@@ -379,9 +382,9 @@ class LlavaOnevisionMultiModalProcessor(
pixel_values_videos = []
for video in videos:
item_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
self.info.get_hf_processor(**hf_kwargs),
dict(text=video_token, **{"videos": video}),
hf_processor_mm_kwargs,
hf_kwargs,
)
pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
@@ -394,8 +397,9 @@ class LlavaOnevisionMultiModalProcessor(
**video_outputs,
)
processed_data = BatchFeature(combined_outputs)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_prompt_updates(
self,
+37 -33
View File
@@ -1556,23 +1556,26 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
class LlavaOnevision2MultiModalProcessor(
BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]
):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_kwargs)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_kwargs)
merged_kwargs.setdefault("return_tensors", "pt")
call_kwargs = {
k: v
@@ -1589,17 +1592,17 @@ class LlavaOnevision2MultiModalProcessor(
"codec_config",
}
}
mm_data = dict(mm_data)
# Explicit None + length checks: ``mm_data[...]`` may be a list, numpy
# array, or tensor, and ``and <array>`` would raise on the ambiguous
# truth value of a multi-element array.
_videos = mm_data.get("videos")
_videos = hf_data.get("videos")
if _videos is not None and not isinstance(_videos, Sized):
raise TypeError(f"object of type '{type(_videos).__name__}' has no len()")
videos_present = _videos is not None and len(_videos) > 0
codec_video_paths = (
_extract_codec_video_paths(mm_data["videos"]) if videos_present else None
_extract_codec_video_paths(hf_data["videos"]) if videos_present else None
)
is_codec_marker = codec_video_paths is not None
# Fallback: caller passed video_backend=codec via mm_processor_kwargs
@@ -1613,7 +1616,7 @@ class LlavaOnevision2MultiModalProcessor(
and call_kwargs.get("video_backend") == "codec"
)
if is_codec_kwarg:
raw = mm_data["videos"]
raw = hf_data["videos"]
if isinstance(raw, str):
codec_video_paths = [raw]
elif isinstance(raw, (list, tuple)) and all(
@@ -1638,7 +1641,7 @@ class LlavaOnevision2MultiModalProcessor(
# Codec backend: HF processor consumes the path string directly and
# performs decode + canvas-packing internally. The dummy ndarray
# we attached during prepare_codec_video_input is discarded here.
mm_data["videos"] = (
hf_data["videos"] = (
codec_video_paths
if len(codec_video_paths) > 1
else codec_video_paths[0]
@@ -1648,11 +1651,9 @@ class LlavaOnevision2MultiModalProcessor(
# ``video_backend="codec"`` via mm_kwargs so the wrapped processor
# dispatches to its codec branch.
output = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(
**{**hf_processor_mm_kwargs, "video_backend": "codec"}
),
dict(text=prompt_text, **mm_data),
{**hf_processor_mm_kwargs, "video_backend": "codec"},
self.info.get_hf_processor(**{**hf_kwargs, "video_backend": "codec"}),
dict(text=prompt_text, **hf_data),
{**hf_kwargs, "video_backend": "codec"},
)
data = dict(output)
processed_data = BatchFeature(
@@ -1660,8 +1661,9 @@ class LlavaOnevision2MultiModalProcessor(
data, codec_video_paths, hf_processor
)
)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
# ---- Frame backend (registered LlavaOnevision2VideoBackend) ------
# Every non-codec video reaches here as a ``(frames_ndarray, metadata)``
@@ -1676,7 +1678,7 @@ class LlavaOnevision2MultiModalProcessor(
# provided by the backend's ``compute_frames_index_to_sample``; SSRF /
# local-file gating is enforced by the connector before decoding.
if videos_present:
timestamp_decimals = hf_processor_mm_kwargs.get(
timestamp_decimals = hf_kwargs.get(
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
)
if not isinstance(
@@ -1690,7 +1692,7 @@ class LlavaOnevision2MultiModalProcessor(
per_video_frames: list[list[Image.Image]] = []
per_video_timestamps: list[list[float]] = []
videos = mm_data["videos"]
videos = hf_data["videos"]
if not isinstance(videos, Iterable):
raise TypeError(f"Unsupported videos input: {type(videos)}")
for item in videos:
@@ -1714,7 +1716,7 @@ class LlavaOnevision2MultiModalProcessor(
# order) -- otherwise mixed image+video requests bind frames to the
# wrong placeholder. ``row_is_video`` labels each grid row so outputs
# can be split back into per-modality keys below.
merged_mm_data = dict(mm_data)
merged_mm_data = dict(hf_data)
existing_images = merged_mm_data.pop("images", None)
caller_images: list[Image.Image] = []
if existing_images:
@@ -1760,9 +1762,9 @@ class LlavaOnevision2MultiModalProcessor(
# **kwargs to the image processor, so passing the full merged kwarg
# set here is a no-op beyond return_tensors/padding.
output = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
self.info.get_hf_processor(**hf_kwargs),
dict(text=new_prompt, **merged_mm_data),
hf_processor_mm_kwargs,
hf_kwargs,
)
data = dict(output)
@@ -1820,20 +1822,22 @@ class LlavaOnevision2MultiModalProcessor(
)
processed_data = BatchFeature(data)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
# ---- Image-only / text-only call --------------------------------
# No videos present: delegate to ``ctx.call_hf_processor``, which
# runs the wrapped processor over the (possibly empty) image set and
# applies float-tensor dtype postprocessing automatically.
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
self.info.get_hf_processor(**hf_kwargs),
dict(text=prompt_text, **hf_data),
hf_kwargs,
)
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
processed_data.update(passthrough_data)
return processed_data
def _rename_codec_outputs_to_video(
self,
+22 -23
View File
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.midashenglm import DashengConfig
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -582,35 +583,33 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
class MiDashengLMMultiModalProcessor(
BaseMultiModalProcessor[MiDashengLMProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
assert isinstance(audios, list)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
min_audio_len = self.info.get_min_audio_len()
processed_audios = [
np.pad(
audio,
(0, min_audio_len - audio.shape[-1]),
mode="constant",
constant_values=0,
)
if isinstance(audio, np.ndarray) and audio.shape[-1] < min_audio_len
else audio
for audio in audios
]
if audios := hf_inputs.hf_data.get("audio"):
assert isinstance(audios, list)
if processed_audios:
mm_data["audio"] = processed_audios
min_audio_len = self.info.get_min_audio_len()
hf_inputs.hf_data["audio"] = [
np.pad(
audio,
(0, min_audio_len - audio.shape[-1]),
mode="constant",
constant_values=0,
)
if isinstance(audio, np.ndarray) and audio.shape[-1] < min_audio_len
else audio
for audio in audios
]
return mm_data, hf_processor_mm_kwargs
return hf_inputs
def _get_mm_fields_config(
self,
+10 -28
View File
@@ -46,6 +46,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.transformers_utils.configs.mimo_v2_omni import Mimo_VLVisionConfig
from vllm.transformers_utils.processors.mimo_v2_omni import (
MiMoOmniProcessor,
@@ -930,27 +931,14 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")
return fields
def _apply_hf_processor_main(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.
Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses
the singular form.
"""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
# Remap audios → audio (MiMoOmniProcessor uses singular param name)
if "audios" in mm_data:
mm_data = {**mm_data, "audio": mm_data["audios"]}
mm_data = {k: v for k, v in mm_data.items() if k != "audios"}
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo."""
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_data = hf_inputs.hf_data
# Handle video_audio items: convert video part to (TCHW, timestamps) tuple
if "video_audio" in mm_data:
@@ -992,7 +980,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
audio=va_item.audio,
)
)
mm_data = {**mm_data, "video_audio": va_converted}
mm_data["video_audio"] = va_converted
if "videos" in mm_data:
converted: list[tuple[torch.Tensor, torch.Tensor]] = []
@@ -1026,15 +1014,9 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
timestamps = torch.arange(T, dtype=torch.float32) / self._INPUT_FPS
converted.append((frames, timestamps))
mm_data = {**mm_data, "videos": converted}
mm_data["videos"] = converted
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
return hf_inputs
def _get_prompt_updates(
self,
+1 -1
View File
@@ -445,7 +445,7 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
if (audios := mm_data.get("audios")) is None:
if (audios := mm_data.get("audio")) is None:
return {}
mm_items = self.info.parse_mm_data({"audio": audios}, validate=False)
+7 -7
View File
@@ -1059,19 +1059,18 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
tokenizer = self.info.get_tokenizer()
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
mm_inputs = self.process_mm_inputs(mm_data, hf_kwargs)
processed_data = BatchFeature(
{
@@ -1079,8 +1078,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
**mm_inputs,
}
)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
mm_data, hf_kwargs, passthrough_data, processed_data
)
def _get_prompt_updates(
self,
+2 -2
View File
@@ -248,7 +248,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
return ImageSize(width=width, height=height)
class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo]):
class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_images = mm_counts.get("image", 0)
@@ -280,7 +280,7 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo])
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+1 -1
View File
@@ -589,7 +589,7 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+21 -20
View File
@@ -1122,6 +1122,9 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
processor = self.info.get_hf_processor()
@@ -1140,28 +1143,28 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
dict(tokens=tokens),
)["input_ids"].tolist()
def _apply_hf_processor_main(
def _call_hf_processor(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_counts = mm_items.get_all_counts()
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not processor_data:
return BatchFeature(dict(passthrough_data))
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
processed_data = self.info.ctx.call_hf_processor(
hf_processor = self.info.get_hf_processor(**hf_kwargs)
return self.info.ctx.call_hf_processor(
hf_processor.process,
dict(text=prompt_text, **processor_data),
hf_processor_mm_kwargs,
hf_data,
hf_kwargs,
)
def _postprocess_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not hf_data:
return processed_data
hf_processor = self.info.get_hf_processor(**hf_kwargs)
tokenizer = hf_processor.tokenizer
image_patch_id = tokenizer.vocab[IMAGE_PATCH_TOKEN]
@@ -1169,7 +1172,7 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
processed_data.pop("input_ids")
if (images := processor_data.get("images")) is not None:
if (images := hf_data.get("images")) is not None:
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [
@@ -1193,8 +1196,6 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
processed_data["num_crops"] = num_crops
processed_data["img_patch_id"] = image_patch_id
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
+23 -18
View File
@@ -1932,6 +1932,9 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
processor = self.info.get_hf_processor()
tokenizer = processor.tokenizer
@@ -1946,19 +1949,20 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_counts = mm_items.get_all_counts()
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
mm_data = dict(processor_data)
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
hf_config = self.info.get_hf_config()
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_kwargs)
def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature:
res = hf_processor(text=text, images=images, videos=videos, **kwargs)
@@ -1973,7 +1977,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
tokenizer = hf_processor.tokenizer
image_processor = hf_processor.image_processor
if videos := mm_data.pop("videos", []):
if videos := hf_data.pop("videos", []):
assert isinstance(videos, Sequence)
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
@@ -1993,16 +1997,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
# NOTE: metadata.frames_indices indicates
# the sampled frames indices of pre-sampled videos, which is
# used to calculate the timestamps. Make sure that
# do_sample_frames in hf_processor_mm_kwargs is false for
# do_sample_frames in hf_kwargs is false for
# presampled videos.
# NOTE: a copy of hf_processor_mm_kwargs is created to update
# NOTE: a copy of hf_kwargs is created to update
# do_sample_frames, otherwise mm_hash for the object will be
# incorrect.
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
video_mm_kwargs = dict(**hf_kwargs)
if "do_sample_frames" not in video_mm_kwargs:
# molmo_utils already has "do_sample_frames" in
# hf_processor_mm_kwargs, don't overwrite it.
# hf_kwargs, don't overwrite it.
video_mm_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", False
)
@@ -2067,11 +2071,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
processed_data = self.info.ctx.call_hf_processor(
patched_call,
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
dict(text=prompt_text, **hf_data),
hf_kwargs,
)
if (images := mm_data.get("images")) is not None:
if (images := hf_data.get("images")) is not None:
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [
@@ -2113,10 +2117,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
)
processed_data.update(all_video_outputs)
processed_data.update(passthrough_data)
processed_data.pop("input_ids")
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+6 -21
View File
@@ -1356,32 +1356,17 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo
class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]):
def _apply_hf_processor_main(
def _call_hf_processor(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
mm_data["audio"] = audios
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
processor_kwargs = _filter_moss_audio_processor_config(hf_processor_mm_kwargs)
processed_data = self.info.ctx.call_hf_processor(
processor_kwargs = _filter_moss_audio_processor_config(dict(hf_kwargs))
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**processor_kwargs),
mm_data,
hf_data,
{},
)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
@@ -66,7 +66,7 @@ from vllm.multimodal.processing import (
PromptUpdateDetails,
cached_encode,
)
from vllm.multimodal.processing.processor import ProcessorInputs
from vllm.multimodal.processing.processor import HFMultiModalInputs, ProcessorInputs
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -465,17 +465,19 @@ class MossTranscribeDiarizeDummyInputsBuilder(
class MossTranscribeDiarizeMultiModalProcessor(
BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
audios = _get_audios_from_mm_data(mm_data)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
if hf_inputs.hf_data:
hf_inputs.hf_data["audio"] = _get_audios_from_mm_data(hf_inputs.hf_data)
return dict(audio=audios), hf_processor_mm_kwargs
return hf_inputs
def _postprocess_hf_mm_data(
self,
+10 -7
View File
@@ -275,16 +275,18 @@ class MuseGlimmerMultiModalProcessor(
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
processor = self.info.get_hf_processor(**hf_kwargs)
tokenizer = processor.tokenizer
config = self.info.get_hf_config()
images = mm_data.get("images", ())
@@ -371,8 +373,9 @@ class MuseGlimmerMultiModalProcessor(
video_feature_sizes=torch.tensor(video_sizes),
)
processed_data = BatchFeature(data=data, tensor_type=None)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
mm_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+23 -8
View File
@@ -70,6 +70,7 @@ from vllm.multimodal.processing import (
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
HFMultiModalInputs,
PromptReplacement,
PromptUpdate,
cached_encode,
@@ -196,6 +197,9 @@ NanoNemotronVLVideoInputs: TypeAlias = (
class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor:
# Not accepted by NanoNemotronVLProcessor.__init__
kwargs.pop("truncation", None)
return self.ctx.init_processor(
NanoNemotronVLProcessor,
config=self.get_hf_config(),
@@ -358,9 +362,23 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
class NanoNemotronVLMultiModalProcessor(
BaseMultiModalProcessor[NanoNemotronVLProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# NanoNemotronVLProcessor expects "audios" instead of "audio"
hf_data = hf_inputs.hf_data
if "audio" in hf_data:
hf_data["audios"] = hf_data.pop("audio")
return hf_inputs
def _get_image_fields_config(self, hf_inputs: BatchFeature):
if self.info.is_dynamic_tiler:
pixel_values_flat = MultiModalFieldConfig.batched("image")
@@ -730,15 +748,12 @@ class NanoNemotronVLMultiModalProcessor(
# Bypass the cached path: the HF processor must receive the
# prompt (with injected <so_embedding>) and the audio data
# together so it can perform audio-token replacement natively.
mm_info = self._apply_hf_processor(inputs, timing_ctx)
prompt_ids = self._postprocess_prompt(inputs.prompt)
mm_res = self._apply_hf_processor(inputs, timing_ctx)
with timing_ctx.record("apply_prompt_updates"):
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
mm_items=mm_items,
prompt_ids=prompt_ids,
mm_kwargs=mm_info.kwargs,
mm_prompt_updates=mm_info.prompt_updates,
mm_res=mm_res,
)
mm_placeholder_ranges = {
@@ -749,8 +764,8 @@ class NanoNemotronVLMultiModalProcessor(
return MultiModalInput(
type="multimodal",
prompt_token_ids=prompt_ids,
mm_kwargs=mm_info.kwargs,
mm_hashes=mm_info.hashes,
mm_kwargs=mm_res.kwargs,
mm_hashes=mm_res.hashes,
mm_placeholders=mm_placeholder_ranges,
)
+3 -3
View File
@@ -326,6 +326,9 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]):
class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def image_indicators_to_visual_tokens(
self,
image_indicators: list[int],
@@ -340,9 +343,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
# -300 is image_atom token, filter them out
return [vte_vocab_size + x + 300 for x in image_indicators if x < -300]
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
+1 -1
View File
@@ -327,7 +327,7 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
if x >= INDICATOR_IDS[0]
]
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+20 -19
View File
@@ -243,36 +243,37 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing
class PaddleOCRVLMultiModalProcessor(
BaseMultiModalProcessor[PaddleOCRVLProcessingInfo]
):
def _apply_hf_processor_main(
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _call_hf_processor(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
final_mm_kwargs = dict(hf_processor_mm_kwargs or {})
final_mm_kwargs = dict(hf_kwargs or {})
final_mm_kwargs.setdefault("images_kwargs", {})
# vLLM use PIL.Image, always set channel_last
final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
processed_data = self.info.ctx.call_hf_processor(
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**final_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
hf_data,
hf_kwargs,
)
def _postprocess_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not hf_data:
return processed_data
num_patches_per_image = processed_data["image_grid_thw"].prod(-1)
processed_data["pixel_values"] = processed_data["pixel_values"].split(
num_patches_per_image.tolist()
)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
+1 -20
View File
@@ -390,28 +390,9 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
input_ids = processed_data["input_ids"]
assert isinstance(input_ids, torch.Tensor)
# Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
# which will cause OverflowError when decoding the prompt_ids.
# Therefore, we need to do an early replacement here
input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID)
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+38 -34
View File
@@ -45,6 +45,7 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
HFMultiModalInputs,
PromptReplacement,
PromptUpdate,
ResolvedPromptUpdate,
@@ -867,40 +868,38 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
def _apply_hf_processor_main(
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# The Phi4MM processor expects "audios" instead of "audio"
hf_data = hf_inputs.hf_data
if "audio" in hf_data:
hf_data["audios"] = hf_data.pop("audio")
sr = self.info.get_feature_extractor(**hf_inputs.hf_kwargs).sampling_rate
if audio_data := hf_data.get("audios", []):
assert isinstance(audio_data, Sequence)
hf_data["audios"] = [(data, sr) for data in audio_data]
return hf_inputs
def _postprocess_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not hf_data:
return processed_data
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
sr = self.info.get_feature_extractor(**hf_processor_mm_kwargs).sampling_rate
raw_audio_data = mm_data.get("audios", [])
assert isinstance(raw_audio_data, Sequence)
audio_data = []
for data in raw_audio_data:
assert isinstance(data, (list, np.ndarray, torch.Tensor))
audio_data.append(data)
processor_data = dict(text=prompt_text, **mm_data)
if audio_data:
processor_data["audios"] = [(data, sr) for data in audio_data]
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
processor_data,
hf_processor_mm_kwargs,
)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_kwargs)
num_img_tokens = [
self.info.get_num_image_tokens(
image_width=img_size[0],
@@ -911,21 +910,26 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
]
processed_data["num_img_tokens"] = num_img_tokens
audio_features = processed_data["input_audio_embeds"]
audios = hf_data.get("audios", [])
assert isinstance(audios, list)
audio_features = processed_data.get("input_audio_embeds")
if audio_features is None:
return processed_data
sr = self.info.get_feature_extractor(**hf_kwargs).sampling_rate
feature_sizes = [
self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data
self.info.get_audio_num_frames(len(audio), sr) for audio, _ in audios
]
processed_data["input_audio_embeds"] = [
audio_features[idx, :size] for idx, size in enumerate(feature_sizes)
]
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
return dict(
input_image_embeds=MultiModalFieldConfig.batched("image"),
+2 -33
View File
@@ -143,39 +143,8 @@ class Phi4SiglipDummyInputsBuilder(
class Phi4SiglipMultiModalProcessor(
BaseMultiModalProcessor[Phi4SiglipProcessingInfo],
):
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
processed = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
# The HF processor's tokenizer_image_token() replaces the "<image>"
# string with IMAGE_TOKEN_INDEX (-200) in input_ids. This breaks
# vLLM's prompt-replacement pipeline which needs to find "<image>"
# as normal sub-tokens. Re-tokenize with the plain tokenizer so
# that "<image>" stays as sub-tokens and can be located by
# PromptReplacement.
# NOTE: tokenizer.__call__() (not .encode()) must be used so that
# added/special tokens like <|user|>, <|end|> are kept as single IDs.
tokenizer = self.info.get_tokenizer()
new_ids = tokenizer(prompt_text).input_ids
processed["input_ids"] = torch.tensor([new_ids])
processed_data = processed
processed_data.update(passthrough_data)
return processed_data
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_mm_fields_config(
self,
+17 -38
View File
@@ -14,7 +14,7 @@ from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
from mistral_common.protocol.instruct.messages import UserMessage
from mistral_common.protocol.instruct.request import ChatCompletionRequest
from packaging.version import Version
from transformers import BatchFeature, PixtralVisionConfig
from transformers import PixtralVisionConfig
from transformers.models.pixtral.image_processing_pixtral import (
_num_image_tokens as _get_pixtral_hf_num_image_tokens,
)
@@ -41,7 +41,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsOptionalItems,
NestedTensors,
)
from vllm.multimodal.parse import (
@@ -53,7 +52,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
MultiModalPromptUpdates,
HFMultiModalInputs,
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
ProcessorInputs,
PromptReplacement,
@@ -67,7 +67,6 @@ from vllm.transformers_utils.processors.pixtral import (
MistralCommonImageProcessor,
MistralCommonPixtralProcessor,
)
from vllm.utils.collection_utils import is_list_of
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -268,21 +267,19 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsOptionalItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_item_counts = mm_items.get_all_counts()
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
mm_placeholders = self._find_mm_placeholders(
prompt_ids,
mm_prompt_updates,
mm_res.prompt_ids,
mm_res.prompt_updates,
)
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
return prompt_ids, mm_placeholders
return mm_res.prompt_ids, mm_placeholders
def _get_mm_fields_config(
self,
@@ -291,35 +288,17 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
) -> Mapping[str, MultiModalFieldConfig]:
return dict(images=MultiModalFieldConfig.batched("image"))
def _apply_hf_processor_main(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# Avoid padding issue
return hf_inputs._replace(
hf_kwargs=dict(hf_inputs.hf_kwargs, return_tensors=None)
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
# Avoid padding issue
dict(**hf_processor_mm_kwargs, return_tensors=None),
)
# Missing batch dimension
if is_list_of(outputs["input_ids"], int):
outputs["input_ids"] = [outputs["input_ids"]]
processed_data = outputs
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates(
self,
@@ -82,6 +82,8 @@ from vllm.multimodal.processing import (
)
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
HFMultiModalInputs,
MultiModalProcessingResult,
MultiModalPromptUpdates,
PlaceholderFeaturesInfo,
PromptReplacement,
@@ -498,6 +500,64 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
class Qwen2_5OmniThinkerMultiModalProcessor(
BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_counts = mm_items.get_all_counts()
if hf_kwargs.get("use_audio_in_video"):
num_videos = mm_counts.get("video", 0)
if mm_counts.get("audio", 0) < num_videos:
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
if num_videos and "text" in hf_inputs.hf_data:
standalone_mm_counts = dict(mm_counts)
standalone_mm_counts["audio"] -= num_videos
hf_inputs.hf_data["text"] = self._get_hf_mm_text(standalone_mm_counts)
hf_data = hf_inputs.hf_data
normalized_kwargs = hf_inputs.hf_kwargs
audios = hf_data.pop("audio", [])
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
if audios:
hf_data["audio"] = audios
elif (
normalized_kwargs.get("use_audio_in_video")
and "audio" not in hf_data
and hf_data.get("videos")
):
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
merged = self.info.ctx.get_merged_mm_kwargs(normalized_kwargs)
if hf_data.get("videos") and (
merged.keys() & {"size", "min_pixels", "max_pixels"}
):
normalized_kwargs = dict(normalized_kwargs)
video_size = dict(self.info.get_hf_processor().video_processor.size)
size_override = merged.get("size")
if size_override is not None:
video_size = video_size | size_override
min_pixels = merged.get("min_pixels")
if min_pixels is not None:
video_size["shortest_edge"] = min_pixels
max_pixels = merged.get("max_pixels")
if max_pixels is not None:
video_size["longest_edge"] = max_pixels
normalized_kwargs["size"] = video_size
return hf_inputs._replace(hf_kwargs=normalized_kwargs)
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -570,12 +630,14 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`."""
mm_item_counts = mm_items.get_all_counts()
prompt_ids = mm_res.prompt_ids
mm_kwargs = mm_res.kwargs
mm_prompt_updates = mm_res.prompt_updates
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
@@ -789,101 +851,33 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
),
]
def _apply_hf_processor_main(
def _postprocess_hf_mm_data(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
"""Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`."""
mm_counts = mm_items.get_all_counts()
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
if use_audio_in_video and "video" in mm_counts:
if mm_counts.get("audio", 0) < mm_counts["video"]:
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
mm_counts["audio"] -= mm_counts["video"]
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
mm_data = dict(processor_data)
audios = mm_data.pop("audios", [])
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
if audios:
# NOTE: Qwen2.5-Omni processor accept "audio"
mm_data["audio"] = audios
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
)
elif (
hf_processor_mm_kwargs.get("use_audio_in_video")
and "audio" not in mm_data
and mm_data.get("videos")
):
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
# and populated mm_data["audio"] itself before delegating here,
# in which case this isn't a genuine "no audio" case. Likewise,
# mm_data can be empty on a multimodal-processor-cache hit, where
# there's nothing to (re-)process this call and the real result
# comes from the cache — not a genuine "no audio" case either.
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
if mm_data.get("videos") and (
merged.keys() & {"size", "min_pixels", "max_pixels"}
):
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
video_size = dict(self.info.get_hf_processor().video_processor.size)
size_override = merged.get("size")
if size_override is not None:
video_size = video_size | size_override
min_pixels = merged.get("min_pixels")
if min_pixels is not None:
video_size["shortest_edge"] = min_pixels
max_pixels = merged.get("max_pixels")
if max_pixels is not None:
video_size["longest_edge"] = max_pixels
hf_processor_mm_kwargs["size"] = video_size
hf_inputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
input_features = hf_inputs.pop("input_features", None)
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
if "input_audio_features" not in hf_inputs and input_features is not None:
input_features = processed_data.pop("input_features", None)
feature_attention_mask = processed_data.get("feature_attention_mask", None)
if "input_audio_features" not in processed_data and input_features is not None:
if feature_attention_mask is not None:
input_features = input_features.permute(0, 2, 1)[
feature_attention_mask.bool()
].permute(1, 0)
hf_inputs["input_audio_features"] = input_features
processed_data["input_audio_features"] = input_features
if (
"audio_feature_lengths" not in hf_inputs
"audio_feature_lengths" not in processed_data
and feature_attention_mask is not None
):
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
processed_data["audio_feature_lengths"] = feature_attention_mask.sum(-1)
video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
video_second_per_grid = processed_data.get("video_second_per_grid", None)
if video_second_per_grid is not None:
hf_inputs["second_per_grid_ts"] = video_second_per_grid
processed_data["second_per_grid_ts"] = video_second_per_grid
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
processed_data = hf_inputs
processed_data.update(passthrough_data)
processed_data.pop("input_ids")
use_audio_in_video = hf_kwargs.get("use_audio_in_video", False)
processed_data["use_audio_in_video"] = torch.tensor(use_audio_in_video)
processed_data.pop("input_ids", None)
return processed_data
+14 -17
View File
@@ -59,6 +59,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -233,29 +234,25 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
mm_data["audio"] = audios
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
truncation=True,
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
return hf_inputs._replace(
hf_kwargs=dict(
hf_inputs.hf_kwargs,
sampling_rate=feature_extractor.sampling_rate,
truncation=True,
)
)
return mm_data, hf_processor_mm_kwargs
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -37,11 +37,10 @@ from vllm.model_executor.models.qwen3_asr import (
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
from vllm.multimodal.inputs import MultiModalKwargsOptionalItems
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing.processor import (
BaseDummyInputsBuilder,
MultiModalPromptUpdates,
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
cached_encode,
)
@@ -114,10 +113,10 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsOptionalItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_kwargs = mm_res.kwargs
audios = mm_kwargs.get("audio", [])
assert len(audios) == 1, (
f"Expected only one audio input for realtime, got {len(audios)}"
@@ -147,7 +146,7 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
# Find the audio_pad token position and expand it to audio_len tokens
expanded_ids = list[int]()
pad_start_idx = -1
for i, tid in enumerate(prompt_ids):
for i, tid in enumerate(mm_res.prompt_ids):
if tid == audio_pad_id and pad_start_idx == -1:
pad_start_idx = i
expanded_ids.extend([audio_pad_id] * audio_len)
@@ -73,6 +73,8 @@ from vllm.multimodal.inputs import (
)
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems
from vllm.multimodal.processing.processor import (
HFMultiModalInputs,
MultiModalProcessingResult,
MultiModalPromptUpdates,
PlaceholderFeaturesInfo,
PromptReplacement,
@@ -1219,13 +1221,26 @@ Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder
class Qwen3OmniMoeThinkerMultiModalProcessor(
Qwen2_5OmniThinkerMultiModalProcessor,
):
def _get_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
audios = processor_data.get("audios")
if mm_items.get_count("audio", strict=False):
# TODO(Isotr0py): Remove this patch after upstream fix PR
# released and Transformers version update:
# https://github.com/huggingface/transformers/pull/41473
hf_inputs = hf_inputs._replace(
hf_kwargs={
**hf_inputs.hf_kwargs,
"audio_kwargs": dict(hf_inputs.hf_kwargs.get("audio_kwargs") or {}),
"text_kwargs": dict(hf_inputs.hf_kwargs.get("text_kwargs") or {}),
}
)
audios = hf_inputs.hf_data.get("audio")
if audios:
feature_extractor = self.info.get_feature_extractor()
hop_length = feature_extractor.hop_length
@@ -1239,47 +1254,30 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
# To make sure the cache works with padding=True, we pre-padded
# the audio to a multiple of hop_length.
processor_data = dict(processor_data)
processor_data["audios"] = [
hf_inputs.hf_data["audio"] = [
pad_to_hop_length(audio)
if isinstance(audio, np.ndarray)
else (pad_to_hop_length(audio[0]), audio[1])
for audio in audios
]
return processor_data, passthrough_data
return hf_inputs
def _apply_hf_processor_main(
def _postprocess_hf_mm_data(
self,
mm_items: MultiModalDataItems,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if mm_items.get_all_counts().get("audio", 0):
# TODO(Isotr0py): Remove this patch after upstream fix PR
# released and Transformers version update:
# https://github.com/huggingface/transformers/pull/41473
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
hf_processor_mm_kwargs["audio_kwargs"] = dict(
hf_processor_mm_kwargs.get("audio_kwargs") or {}
)
hf_processor_mm_kwargs["text_kwargs"] = dict(
hf_processor_mm_kwargs.get("text_kwargs") or {}
)
processed_data = super()._apply_hf_processor_main(
mm_items,
hf_processor_mm_kwargs,
processed_data = super()._postprocess_hf_mm_data(
mm_data, hf_processor_mm_kwargs, processed_data
)
if (
"audio_feature_lengths" in processed_data
and "feature_attention_mask" in processed_data
):
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, _ = self._get_hf_mm_data(valid_mm_items)
audios = processor_data.get("audios", [])
audios = mm_data.get("audio", [])
if audios:
feature_extractor = self.info.get_feature_extractor(
**hf_processor_mm_kwargs
@@ -1312,12 +1310,14 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""Qwen3-Omni reimplements this function to handle `use_audio_in_video`."""
mm_item_counts = mm_items.get_all_counts()
prompt_ids = mm_res.prompt_ids
mm_kwargs = mm_res.kwargs
mm_prompt_updates = mm_res.prompt_updates
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
use_audio_in_video = False
+16 -22
View File
@@ -1327,27 +1327,25 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
return proc_impl is not None and proc_impl is not mixin_impl
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data)
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
# Separate video processing from image processing. Because the videos
# are processed into several image patches
video_input_ids_lst: list[list[int]] = []
if videos := mm_data.pop("videos", []):
if videos := hf_data.pop("videos", []):
video_grid_thw_lst = []
pixel_values_videos_lst = []
timestamps_per_video = []
@@ -1370,10 +1368,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# NOTE: a copy of is created to update do_sample_frames,
# otherwise mm_hash for the object will be incorrect.
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(
hf_processor_mm_kwargs, modality="video"
)
video_mm_kwargs = dict(**hf_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
video_size = dict(self.info.get_video_processor().size)
size_override = merged.get("size")
@@ -1485,13 +1481,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# fps/num_frames are video-only kwargs already consumed by the loop;
# exclude them so the text/image processor call below never gets a list.
non_video_mm_kwargs = {
k: v
for k, v in hf_processor_mm_kwargs.items()
if k not in ("fps", "num_frames")
k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames")
}
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**non_video_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_data,
non_video_mm_kwargs,
)
@@ -1516,9 +1510,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
processed_data["input_ids"] = [expanded_ids]
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+3 -1
View File
@@ -202,7 +202,9 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
with timing_ctx.record("apply_hf_processor"):
_, passthrough_data = self._get_hf_mm_data(mm_items)
passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_processor_mm_kwargs
).passthrough_data
mm_processed_data = BatchFeature(
{
k: torch.as_tensor(v).unsqueeze(0)
@@ -219,18 +219,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
Subclasses add the strategy for locating placeholders in the prompt.
"""
def _get_hf_mm_data(
self,
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""Rename the parser's `audios` key to the `audio` argument HF audio
processors take.
"""
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
if self.info._is_audio_model() and "audios" in processor_data:
processor_data["audio"] = processor_data.pop("audios")
return processor_data, passthrough_data
def _get_modality_field_names(self, modality: str) -> set[str]:
"""Names of the fields the sub-processor for `modality` produces."""
# TODO: use else branch only once huggingface/transformers#44394 lands.
@@ -353,6 +341,9 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
return mm_fields
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _unpad_audios(
self,
hf_inputs: "BatchFeature",
@@ -597,21 +588,31 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
# work for Transformers. The vision path has logic tied to
# `mm_tokens_per_modality` in _apply_vision()
hf_processor_mm_kwargs = {
**hf_processor_mm_kwargs,
# vLLM needs the untruncated sequence to keep placeholder
# tokens aligned. Note that the text inputs are just dummy
# text, not the original prompt. The original prompt is
# already tokenized by the renderer.
"truncation": False,
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
# (the HF processor call disables `add_special_tokens`).
"add_special_tokens": False,
**hf_processor_mm_kwargs,
}
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
processor_data, _, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_processor_mm_kwargs
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
# The real prompt is fed to the processor below and the placeholder
# ranges are read back out of its output, so the dummy text meant for
# media-only callers is discarded here.
processor_data.pop("text", None)
has_mm_data = any(processor_data.values())
# Ask which modality owns each token of the expanded prompt, which
# is the only marking of the tokens an expansion adds around an item
if any(processor_data.values()):
if has_mm_data:
processor_data = {**processor_data, "return_mm_token_type_ids": True}
try:
@@ -621,7 +622,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
hf_processor_mm_kwargs,
)
except ValueError:
if any(processor_data.values()):
if has_mm_data:
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
@@ -792,31 +793,37 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
except (AttributeError, KeyError, TypeError):
return None
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> "BatchFeature":
mm_counts = mm_items.get_all_counts()
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
# Ask for the replacement each placeholder expands to, and record it as
# per-item fields: its token ids, and the tokens or patches behind them
if has_mm_data := any(mm_data.values()):
mm_data = {**mm_data, "return_text_replacement_offsets": True}
if has_mm_data := any(hf_data.values()):
hf_data = {**hf_data, "return_text_replacement_offsets": True}
try:
hf_inputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
self.info.get_hf_processor(**hf_kwargs),
dict(text=prompt_text, **hf_data),
hf_kwargs,
)
except ValueError:
if any(mm_data.values()):
if any(hf_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
@@ -825,7 +832,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)
self._unpad_audios(hf_inputs, hf_data, hf_kwargs)
# Drop the inputs the model would reject
hf_inputs.pop("mm_token_type_ids", None)
@@ -845,9 +852,10 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"so it can be fixed, and install `transformers<5.15.0` in the "
"meantime to locate placeholders in the expanded prompt instead."
)
hf_inputs.update(passthrough_data)
hf_inputs.pop("input_ids", None)
return hf_inputs
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, hf_inputs
)
tokenizer = self.info.get_tokenizer()
replacements = defaultdict[str, list[list[int]]](list)
@@ -865,7 +873,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
)
if modality == "image":
hf_inputs["num_image_patches"] = self._get_num_image_patches(
hf_inputs, mm_data, len(seqs)
hf_inputs, hf_data, len(seqs)
)
elif modality == "audio":
counts = []
@@ -874,10 +882,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
hf_inputs.update(passthrough_data)
hf_inputs.pop("input_ids")
return hf_inputs
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, hf_inputs
)
# From this version on, a processor reporting no offsets is an error rather than a
+21 -60
View File
@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
cached_encode,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.renderers import TokenizeParams
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
@@ -190,72 +191,32 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
audios = mm_data.get("audios", [])
assert isinstance(audios, list)
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
include_audio_num_chunks=True,
)
return mm_data, hf_processor_mm_kwargs
def _apply_hf_processor_main(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
if processor_data:
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
processor_data,
hf_processor_mm_kwargs,
# The Ultravox processor accepts "audios" instead of "audio"
hf_data = hf_inputs.hf_data
if "audio" in hf_data:
hf_data["audios"] = hf_data.pop("audio")
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
normalized_kwargs = dict(hf_inputs.hf_kwargs)
# The remote processor already hardcodes truncation=False and forwards
# its kwargs to the audio processor, where another value would collide.
normalized_kwargs.pop("truncation", None)
return hf_inputs._replace(
hf_kwargs=dict(
normalized_kwargs,
sampling_rate=feature_extractor.sampling_rate,
include_audio_num_chunks=True,
)
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
if prompt_text is not None:
processor_data = dict(text=prompt_text, **processor_data)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
def patched_call(**kwargs) -> BatchFeature:
# The remote UltravoxProcessor does not support the
# `truncation` kwarg injected by `call_hf_processor`: it
# passes `truncation` to its audio processor by itself while
# also forwarding `**kwargs` there, causing a duplicate
# keyword argument error. Its hardcoded `truncation=False`
# already matches vLLM's intended behavior.
kwargs.pop("truncation", None)
return hf_processor(**kwargs)
processed_data = self.info.ctx.call_hf_processor(
patched_call,
processor_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
else:
processed_data = BatchFeature(dict(passthrough_data))
return self._postprocess_hf_mm_data(
processor_data,
hf_processor_mm_kwargs,
processed_data,
)
def _postprocess_hf_mm_data(
+2 -2
View File
@@ -75,7 +75,7 @@ from vllm.multimodal.parse import (
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
from vllm.multimodal.processing.context import TimingContext
from vllm.multimodal.processing.inputs import ProcessorInputs
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
from vllm.multimodal.processing.processor import MultiModalProcessingResult
from vllm.transformers_utils.processors.deepseek_ocr import (
BASE_SIZE,
CROP_MODE,
@@ -225,7 +225,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
# The processor logic differs for single-image (crop) vs multi-image
# (no crop) requests. The processing cache assumes per-item output is
# invariant of how many images are passed per prompt, so we only cache
+18 -39
View File
@@ -16,7 +16,7 @@ from mistral_common.protocol.instruct.messages import UserMessage
from mistral_common.protocol.instruct.request import ChatCompletionRequest
from mistral_common.protocol.transcription.request import TranscriptionRequest
from mistral_common.tokens.tokenizers.audio import Audio
from transformers import BatchFeature, WhisperConfig
from transformers import WhisperConfig
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
@@ -36,7 +36,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItems,
MultiModalKwargsOptionalItems,
NestedTensors,
)
from vllm.multimodal.parse import (
@@ -48,7 +47,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
MultiModalPromptUpdates,
HFMultiModalInputs,
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
ProcessorInputs,
PromptReplacement,
@@ -61,7 +61,6 @@ from vllm.transformers_utils.processors.voxtral import (
MistralCommonFeatureExtractor,
MistralCommonVoxtralProcessor,
)
from vllm.utils.collection_utils import is_list_of
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription
from .utils import init_vllm_registered_model, maybe_prefix
@@ -209,21 +208,19 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsOptionalItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_item_counts = mm_items.get_all_counts()
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
mm_placeholders = self._find_mm_placeholders(
prompt_ids,
mm_prompt_updates,
mm_res.prompt_ids,
mm_res.prompt_updates,
)
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
return prompt_ids, mm_placeholders
return mm_res.prompt_ids, mm_placeholders
def _get_mm_fields_config(
self,
@@ -241,38 +238,20 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
# skip validation here
pass
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
# MistralCommonVoxtralProcessor accepts "audio"
mm_data["audio"] = audios
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# Avoid padding issue
hf_processor_mm_kwargs = dict(**hf_processor_mm_kwargs, return_tensors=None)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
# Missing batch dimension
input_ids = processed_data.get("input_ids")
if is_list_of(input_ids, int):
processed_data["input_ids"] = [input_ids]
return processed_data
return hf_inputs._replace(
hf_kwargs=dict(hf_inputs.hf_kwargs, return_tensors=None)
)
def _get_prompt_updates(
self,
@@ -28,11 +28,10 @@ from vllm.model_executor.models.voxtral import (
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
from vllm.multimodal.inputs import MultiModalKwargsOptionalItems
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
MultiModalPromptUpdates,
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
)
from vllm.sequence import IntermediateTensors
@@ -60,10 +59,10 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsOptionalItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_kwargs = mm_res.kwargs
# there are no placeholder audio tokens for streaming
# so we need to build the place placeholder positions manually
@@ -86,7 +85,7 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
* [0], # only used for length computation, so we can take dummy inputs
is_embed=None,
)
return prompt_ids, {"audio": [features_info]}
return mm_res.prompt_ids, {"audio": [features_info]}
class TimeEmbedding(torch.nn.Module):
+15 -27
View File
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.renderers import TokenizeParams
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.jsontree import json_map_leaves
@@ -743,6 +744,9 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def create_encoder_prompt(
self,
prompt: list[int],
@@ -754,37 +758,21 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
# for encoder profiling.
return [0]
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
return hf_inputs._replace(
hf_kwargs=dict(
hf_inputs.hf_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+24 -16
View File
@@ -34,6 +34,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
from .video import preprocess_dots3_note_video
@@ -630,26 +631,33 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingInfo]):
def _get_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
if "video" not in mm_items:
return processor_data, passthrough_data
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
videos = mm_items.get_items("video", VideoProcessorItems)
raw_videos: list[object] = []
for index, item in enumerate(videos.data):
if isinstance(item, MediaWithBytes):
raw_videos.append(item.original_bytes)
else:
raw_videos.append(videos.get(index))
processor_data = dict(processor_data)
processor_data["videos"] = raw_videos
return processor_data, passthrough_data
# The Dots3Note processor accepts "audios" instead of "audio"
hf_data = hf_inputs.hf_data
if "audio" in hf_data:
hf_data["audios"] = hf_data.pop("audio")
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
if "video" in mm_items:
videos = mm_items.get_items("video", VideoProcessorItems)
raw_videos: list[object] = []
for index, item in enumerate(videos.data):
if isinstance(item, MediaWithBytes):
raw_videos.append(item.original_bytes)
else:
raw_videos.append(videos.get(index))
hf_data["videos"] = raw_videos
return hf_inputs
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_mm_fields_config(
+8 -8
View File
@@ -182,20 +182,19 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
processor = self.info.get_hf_processor(**hf_kwargs)
tokenizer = self.info.get_tokenizer()
images = mm_data.get("images") or []
audios = mm_data.get("audios") or []
audios = mm_data.get("audio") or []
if not isinstance(images, list):
images = list(cast(Iterable[Any], images))
if not isinstance(audios, list):
@@ -235,8 +234,9 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)
processed_data = BatchFeature(data=data, tensor_type=None)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
mm_data, hf_kwargs, passthrough_data, processed_data
)
def _tokenize_with_placeholders(
self,
+25 -25
View File
@@ -32,6 +32,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.multimodal.video import (
VIDEO_LOADER_REGISTRY,
VideoBackend,
@@ -307,26 +308,21 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing
class MiniMaxM3VLMultiModalProcessor(
BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo]
):
def _apply_hf_processor_main(
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
hf_data = hf_inputs.hf_data
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data)
# With ``video_needs_metadata=True`` each video arrives as a
# ``(frames, metadata)`` tuple. Split the frames back out and forward the
# metadata as ``VideoMetadata`` so the processor emits timestamps.
videos = cast(list | None, mm_data.get("videos"))
videos = cast(list | None, hf_data.get("videos"))
video_metadata: list[VideoMetadata] | None = None
if videos:
frames_only = []
@@ -346,21 +342,25 @@ class MiniMaxM3VLMultiModalProcessor(
# stays consistent with _get_prompt_updates.
meta.setdefault("total_num_frames", len(frames))
video_metadata.append(VideoMetadata(**meta))
mm_data["videos"] = frames_only
hf_data["videos"] = frames_only
# Override the video processor's default do_resize=False (set for a
# pre-resized pipeline) to True for vLLM's raw-frame inputs.
merged = dict(do_resize=True, **hf_processor_mm_kwargs)
data = dict(text=prompt_text, **mm_data)
if video_metadata is not None:
data["video_metadata"] = video_metadata
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
data,
hf_data["video_metadata"] = video_metadata
return hf_inputs
def _call_hf_processor(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override the video processor's default for vLLM's raw-frame inputs.
merged = dict(do_resize=True, **hf_kwargs)
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_kwargs),
hf_data,
merged,
)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
+2 -8
View File
@@ -321,12 +321,6 @@ class InputProcessingContext:
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
# vLLM needs the full untruncated sequence to keep multi-modal
# placeholder tokens aligned; note that the text inputs in
# call_hf_processor are just dummy text, not the original prompt.
# The original prompt is already tokenized by the renderer.
merged_kwargs.setdefault("truncation", False)
allowed_kwargs = get_allowed_kwarg_only_overrides(
hf_processor,
merged_kwargs,
@@ -426,7 +420,7 @@ class BaseProcessingInfo:
def get_data_parser(self) -> MultiModalDataParser:
"""Constructs a parser to preprocess multi-modal data items
before passing them to
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
[`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].
You can support additional modalities by creating a subclass
of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
@@ -506,7 +500,7 @@ class BaseProcessingInfo:
"""Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
before passing them to
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
[`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].
"""
mm_items = self.data_parser.parse_mm_data(mm_data)
+116 -84
View File
@@ -1008,7 +1008,20 @@ For an item `MultiModalPromptUpdates[k][i]`,
_I = TypeVar("_I", bound=BaseProcessingInfo)
class MultiModalProcessingInfo(NamedTuple):
class HFMultiModalInputs(NamedTuple):
hf_data: dict[str, object]
"""The data inputs to the HF processor."""
hf_kwargs: Mapping[str, object]
"""The non-data inputs to the HF processor."""
passthrough_data: dict[str, object]
"""The inputs to assign to the output of the HF processor,
instead of passing them to the HF processor."""
class MultiModalProcessingResult(NamedTuple):
prompt_ids: list[int]
kwargs: MultiModalKwargsOptionalItems
hashes: MultiModalHashes
prompt_updates: MultiModalPromptUpdates
@@ -1137,23 +1150,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
return find_mm_placeholders(new_token_ids, mm_prompt_updates)
def _get_hf_mm_data(
self,
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""Extract processor and passthrough data from multi-modal items."""
processor_data = dict[str, object]()
passthrough_data = dict[str, object]()
for items in mm_items.values():
processor_data.update(items.get_processor_data())
passthrough_data.update(items.get_passthrough_data())
return processor_data, passthrough_data
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None:
"""Get the text to pass to the HF processor alongside the multi-modal
data.
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
"""Get the text to pass to the HF processor alongside the multi-modal data.
By default, no text is passed. If the HF processor requires that
text and multi-modal items correspond to each other, you should
@@ -1162,29 +1160,53 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"""
return None
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""Pre-process the multi-modal data and HF processor keyword arguments
before they are passed to the HF processor.
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
"""Extract the multi-modal data and corresponding keyword arguments
for HF processing.
By default, both are returned as-is. If the HF processor expects the
multi-modal data under different keys than those provided by the
multi-modal items (e.g. `audio` instead of `audios`), or requires
additional keyword arguments (e.g. `sampling_rate`), you should
override this method.
If the HF processor expects the multi-modal data under different keys
than those provided by the multi-modal items, or requires additional
keyword arguments, you should override this method.
"""
return mm_data, hf_processor_mm_kwargs
hf_data = dict[str, object]()
passthrough_data = dict[str, object]()
for items in mm_items.values():
if not items:
continue
hf_data.update(items.get_processor_data())
passthrough_data.update(items.get_passthrough_data())
if hf_data:
prompt_text = self._get_hf_mm_text(mm_items.get_all_counts())
if prompt_text is not None:
hf_data = dict(text=prompt_text, **hf_data)
# vLLM needs the untruncated sequence to keep placeholder
# tokens aligned. Note that the text inputs are just dummy
# text, not the original prompt. The original prompt is
# already tokenized by the renderer.
hf_kwargs = {"truncation": False, **hf_kwargs}
# HF processors accept (images, text, videos, audio)
# so we need to remap vLLM's "audios" -> HF "audio"
if "audios" in hf_data:
hf_data["audio"] = hf_data.pop("audios")
return HFMultiModalInputs(hf_data, hf_kwargs, passthrough_data)
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
mm_inputs: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
"""Post-process the output of the HF processor.
"""Post-process the combined processor and passthrough data.
By default, the output is returned as-is. If you need to modify the
output of the HF processor before it is converted into multi-modal
@@ -1192,41 +1214,51 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"""
return processed_data
def _call_hf_processor(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Call the HF processor on the normalized multi-modal inputs."""
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_kwargs),
hf_data,
hf_kwargs,
)
def _finalize_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
passthrough_data: Mapping[str, object],
processed_data: BatchFeature | None = None,
) -> BatchFeature:
"""Merge passthrough fields and apply model-specific postprocessing."""
if processed_data is None:
from transformers.feature_extraction_utils import BatchFeature
processed_data = BatchFeature()
processed_data.update(passthrough_data)
return self._postprocess_hf_mm_data(hf_data, hf_kwargs, processed_data)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Apply the HF processor on the multi-modal data."""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if processor_data:
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
processor_data, hf_processor_mm_kwargs
)
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
if prompt_text is not None:
processor_data = dict(text=prompt_text, **processor_data)
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
processor_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
if hf_data:
processed_data = self._call_hf_processor(hf_data, hf_kwargs)
else:
from transformers.feature_extraction_utils import BatchFeature
processed_data = None
processed_data = BatchFeature(dict(passthrough_data))
return self._postprocess_hf_mm_data(
processor_data,
hf_processor_mm_kwargs,
processed_data,
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
@@ -1261,6 +1293,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_missing_data = {}
for modality, idxs in mm_missing_idxs.items():
if not idxs:
continue
missing_modality_data = []
for idx in idxs:
data = mm_data_items[modality][idx]
@@ -1271,6 +1306,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
)
else:
missing_modality_data.append(data)
mm_missing_data[modality] = missing_modality_data
mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False)
@@ -1342,11 +1378,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
with timing_ctx.record("apply_hf_processor"):
mm_processed_data = self._apply_hf_processor_main(
mm_items=inputs.mm_data_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
hf_kwargs=inputs.hf_processor_mm_kwargs,
)
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
@@ -1369,26 +1405,28 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_kwargs,
)
mm_info = MultiModalProcessingInfo(
return MultiModalProcessingResult(
prompt_ids=self._postprocess_prompt(inputs.prompt),
kwargs=mm_kwargs,
hashes=mm_hashes,
prompt_updates=mm_prompt_updates,
)
return mm_info
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
"""Apply the HF processor on the full prompt text,
caching the results and reusing cached results.
"""
cache = self.cache
has_passthrough_data = any(
len(items.get_passthrough_data()) > 0
for items in inputs.mm_data_items.values()
)
_, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items)
if cache is None or passthrough_data:
if cache is None or has_passthrough_data:
return self._apply_hf_processor(inputs, timing_ctx)
with timing_ctx.record("get_mm_hashes"):
@@ -1410,7 +1448,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
with timing_ctx.record("apply_hf_processor"):
mm_missing_processed_data = self._apply_hf_processor_main(
mm_items=mm_missing_data_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
hf_kwargs=inputs.hf_processor_mm_kwargs,
)
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
@@ -1435,14 +1473,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_missing_prompt_updates=mm_missing_prompt_updates,
)
mm_info = MultiModalProcessingInfo(
return MultiModalProcessingResult(
prompt_ids=self._postprocess_prompt(inputs.prompt),
kwargs=mm_kwargs,
hashes=mm_hashes,
prompt_updates=mm_prompt_updates,
)
return mm_info
def _apply_token_matches(
self,
prompt: list[int],
@@ -1670,17 +1707,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsOptionalItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_item_counts = mm_items.get_all_counts()
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
prompt_ids, mm_placeholders = self._apply_prompt_updates(
prompt_ids,
mm_prompt_updates,
mm_res.prompt_ids,
mm_res.prompt_updates,
)
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
@@ -1705,15 +1740,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
4. Extract information about the placeholder tokens from the
processed token IDs.
"""
prompt_ids = self._postprocess_prompt(inputs.prompt)
mm_info = self._cached_apply_hf_processor(inputs, timing_ctx)
mm_res = self._cached_apply_hf_processor(inputs, timing_ctx)
with timing_ctx.record("apply_prompt_updates"):
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
mm_items=inputs.mm_data_items,
prompt_ids=prompt_ids,
mm_kwargs=mm_info.kwargs,
mm_prompt_updates=mm_info.prompt_updates,
mm_res=mm_res,
)
mm_placeholder_ranges = {
@@ -1723,8 +1755,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
return mm_input(
prompt_token_ids=prompt_ids,
mm_kwargs=mm_info.kwargs,
mm_hashes=mm_info.hashes,
mm_kwargs=mm_res.kwargs,
mm_hashes=mm_res.hashes,
mm_placeholders=mm_placeholder_ranges,
)