[MM] Further cleanup _apply_hf_processor_main (#53610)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -454,17 +454,17 @@ return a schema of the tensors outputted by the HF processor that are related to
|
||||
??? code
|
||||
|
||||
```python
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
# Mistral3Processor requires text corresponding to the images
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
if not hf_data:
|
||||
return processed_data
|
||||
|
||||
pixel_values = processed_data.get("pixel_values")
|
||||
@@ -486,25 +486,36 @@ return a schema of the tensors outputted by the HF processor that are related to
|
||||
calls the HF processor on the multi-modal data without passing any text.
|
||||
If the HF processor instead requires text corresponding to the multi-modal items,
|
||||
you should override
|
||||
[_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text]
|
||||
[_get_hf_mm_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_text]
|
||||
to return the dummy text from
|
||||
[BaseDummyInputsBuilder.get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text]
|
||||
like in the example above. If the HF processor expects the multi-modal data
|
||||
under different keys than those provided by the multi-modal items
|
||||
(e.g. `audio` instead of `audios`), or requires additional keyword arguments
|
||||
under different keys than those provided by the multi-modal items,
|
||||
or requires additional keyword arguments
|
||||
(e.g. `sampling_rate`), you should override
|
||||
[_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data].
|
||||
[_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].
|
||||
If you need to modify the output of the HF processor,
|
||||
you should override
|
||||
[_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data].
|
||||
Passthrough fields have already been merged into `processed_data` when this
|
||||
hook is called.
|
||||
If you need to change how the normalized inputs are passed to the HF
|
||||
processor (for example, to call a method other than `__call__`, or to use
|
||||
different kwargs to construct and invoke it), override
|
||||
[_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor].
|
||||
If the HF processor applies additional transformations to the prompt tokens
|
||||
that are not reflected in its multi-modal outputs (e.g. prepending a BOS token),
|
||||
you should override
|
||||
[_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt]
|
||||
to replicate them.
|
||||
For even more control over how the HF processor is called, you can override
|
||||
Only processors that require multiple HF calls or manually construct the
|
||||
complete output should override
|
||||
[_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]
|
||||
directly.
|
||||
directly. Such implementations must obtain their normalized inputs through
|
||||
[_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs]
|
||||
and return through
|
||||
[_finalize_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._finalize_hf_mm_data]
|
||||
so passthrough fields and model-specific postprocessing are applied consistently.
|
||||
|
||||
Since `pixel_values` is now a list with one tensor per image, we can override
|
||||
[_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows:
|
||||
@@ -724,7 +735,7 @@ Examples:
|
||||
|
||||
### Custom HF processor
|
||||
|
||||
Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. It is then applied to the multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor] inside [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main].
|
||||
Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. The default [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor] applies it to the normalized multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor]. Override `_call_hf_processor` if the custom processor instead exposes a different callable interface.
|
||||
|
||||
Examples:
|
||||
|
||||
|
||||
@@ -8,9 +8,16 @@ In vLLM's rendering pipeline (see [BaseRenderer][vllm.renderers.base.BaseRendere
|
||||
|
||||
Since Transformers 5.10, `ProcessorMixin` now allows multi-modal inputs to be passed by themselves. However, certain subclasses (such as `ChameleonProcessor`) and older out-of-tree implementations may still define their own `__call__` method that assumes the presence of text with corresponding placeholder tokens. This causes a problem as we don't have the original text anymore to pass to these processors.
|
||||
|
||||
To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text] returns so that [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] passes it to the HF processor together with the multi-modal inputs to obtain the processed multi-modal data.
|
||||
To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_mm_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_text] returns. [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs] adds this text and `truncation=False` to the normalized HF inputs. The truncation setting is attached only to calls that use dummy text, so unrelated processor calls do not receive a text-specific option.
|
||||
|
||||
Similarly, since the multi-modal data extracted by vLLM may not match what a specific HF processor expects, [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] allows each model to adapt the inputs via [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data] (e.g. renaming keys like `audios` to `audio` or injecting extra keyword arguments such as `sampling_rate`) and the outputs via [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data], without having to reimplement the entire method.
|
||||
HF processing is split into four overridable stages:
|
||||
|
||||
1. [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs] extracts and normalizes processor data, kwargs, and passthrough data. It centrally renames vLLM's `audios` field to the standard HF `audio` keyword; processors that require another spelling can adapt it in an override.
|
||||
2. [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor] invokes the HF processor. Models can override it when they use a different callable or need separate construction and invocation kwargs.
|
||||
3. [_finalize_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._finalize_hf_mm_data] merges passthrough fields into the processed output. It lazily creates an empty output when there is no processor data.
|
||||
4. [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data] performs model-specific output transformations after passthrough fields have been merged.
|
||||
|
||||
This division lets most models customize one stage without reimplementing [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]. Implementations that genuinely require multiple processor calls or manually construct their complete output may still override that method, but must use `_get_hf_mm_inputs` and `_finalize_hf_mm_data` to retain the common input and output behavior.
|
||||
|
||||
## Prompt Update Detection
|
||||
|
||||
|
||||
@@ -374,21 +374,9 @@ class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):
|
||||
class AudioFlamingo3MultiModalProcessor(
|
||||
BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo]
|
||||
):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", None)
|
||||
if audios is not None:
|
||||
mm_data["audio"] = audios
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
|
||||
@@ -119,7 +119,7 @@ class BailingMoeV3VLDummyInputsBuilder(
|
||||
class BailingMoeV3VLMultiModalProcessor(
|
||||
BaseMultiModalProcessor[BailingMoeV3VLProcessingInfo]
|
||||
):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _apply_prompt_updates(
|
||||
|
||||
@@ -222,7 +222,7 @@ class Cohere2VisionDummyInputsBuilder(
|
||||
class Cohere2VisionMultiModalProcessor(
|
||||
BaseMultiModalProcessor[Cohere2VisionProcessingInfo]
|
||||
):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import Any, ClassVar
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers import BatchFeature, PretrainedConfig
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
@@ -50,7 +50,10 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
TimingContext,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
|
||||
from vllm.multimodal.processing.processor import (
|
||||
HFMultiModalInputs,
|
||||
MultiModalProcessingResult,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.processors.cohere_asr import (
|
||||
@@ -1962,42 +1965,29 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
|
||||
) -> list[int]:
|
||||
return [0]
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
|
||||
return hf_inputs._replace(
|
||||
hf_kwargs=dict(
|
||||
hf_inputs.hf_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
)
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "labels" in processed_data:
|
||||
processed_data["input_ids"] = processed_data.pop("labels")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
) -> MultiModalProcessingResult:
|
||||
# Dithering injects noise into the extracted features, so the
|
||||
# feature extractor is not a pure function of its input. Since the
|
||||
# processing cache assumes that processor outputs are invariant
|
||||
|
||||
@@ -157,15 +157,14 @@ class ColModernVBertMultiModalProcessor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
@@ -193,8 +192,9 @@ class ColModernVBertMultiModalProcessor(
|
||||
result.update(image_outputs)
|
||||
|
||||
processed_data = result
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
mm_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -20,7 +20,7 @@ from collections.abc import Iterable, Mapping
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import BatchFeature, PaliGemmaProcessor
|
||||
from transformers import PaliGemmaProcessor
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
|
||||
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
|
||||
from .interfaces import SupportsLateInteraction
|
||||
from .interfaces_base import default_pooling_type
|
||||
@@ -59,31 +60,19 @@ class ColPaliProcessingInfo(PaliGemmaProcessingInfo):
|
||||
class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
|
||||
"""Multimodal processor for ColPali."""
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
# The ColPali tokenizer_config.json ships with a small default
|
||||
# max_length (50) that truncates the 1024 image tokens inserted
|
||||
# by PaliGemmaProcessor, causing a token-count mismatch.
|
||||
# vLLM enforces its own max_model_len, so we disable HF
|
||||
# truncation to keep all image + text tokens intact.
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
mm_data,
|
||||
dict(**hf_processor_mm_kwargs, truncation=False),
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return hf_inputs._replace(hf_kwargs=dict(hf_inputs.hf_kwargs, truncation=False))
|
||||
|
||||
|
||||
@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
|
||||
|
||||
@@ -46,10 +46,10 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import (
|
||||
BASE_SIZE,
|
||||
@@ -287,34 +287,19 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing
|
||||
class DeepseekOCRMultiModalProcessor(
|
||||
BaseMultiModalProcessor[DeepseekOCRProcessingInfo]
|
||||
):
|
||||
def _apply_hf_processor_main(
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
if "text" in hf_inputs.hf_data:
|
||||
hf_inputs.hf_data["prompt"] = hf_inputs.hf_data.pop("text")
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
if mm_data:
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(prompt=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
else:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
processed_data = tokenizer(
|
||||
prompt_text, add_special_tokens=True, return_tensors="pt"
|
||||
)
|
||||
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return hf_inputs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -45,9 +45,9 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import (
|
||||
BASE_SIZE,
|
||||
@@ -158,35 +158,19 @@ class DeepseekOCR2DummyInputsBuilder(
|
||||
class DeepseekOCR2MultiModalProcessor(
|
||||
BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo]
|
||||
):
|
||||
def _apply_hf_processor_main(
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
if "text" in hf_inputs.hf_data:
|
||||
hf_inputs.hf_data["prompt"] = hf_inputs.hf_data.pop("text")
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
if mm_data:
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(prompt=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
else:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
processed_data = tokenizer(
|
||||
prompt_text, add_special_tokens=True, return_tensors="pt"
|
||||
)
|
||||
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
return hf_inputs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
MultiModalProcessingInfo,
|
||||
MultiModalProcessingResult,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
@@ -233,7 +233,7 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
|
||||
class DeepseekVL2MultiModalProcessor(
|
||||
BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]
|
||||
):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
@@ -307,7 +307,7 @@ class DeepseekVL2MultiModalProcessor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
) -> MultiModalProcessingResult:
|
||||
# The processor logic is different for len(images) <= 2 vs > 2
|
||||
# Since the processing cache assumes that the processor output is
|
||||
# invariant of how many images are passed per prompt, we only
|
||||
|
||||
@@ -71,6 +71,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -1112,34 +1113,32 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
pixel_values = pixel_values.to(hf_config.dtype)
|
||||
return pixel_values
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
hf_data = hf_inputs.hf_data
|
||||
|
||||
if "images" not in mm_data and "videos" not in mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
if "images" not in hf_data and "videos" not in hf_data:
|
||||
return hf_inputs
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
if "images" not in mm_data:
|
||||
mm_data["images"] = []
|
||||
if "videos" not in mm_data:
|
||||
mm_data["videos"] = []
|
||||
prompt_text = hf_data["text"]
|
||||
hf_data["text"] = [prompt_text]
|
||||
hf_data.setdefault("images", [])
|
||||
hf_data.setdefault("videos", [])
|
||||
|
||||
# Check if HF processor supports video metadata
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_inputs.hf_kwargs)
|
||||
supports_video_metadata = getattr(
|
||||
hf_processor, "supports_video_metadata", False
|
||||
)
|
||||
|
||||
videos = mm_data["videos"]
|
||||
videos = hf_data["videos"]
|
||||
assert isinstance(videos, Sequence)
|
||||
if videos and not supports_video_metadata:
|
||||
# Old HF processor, unwrap tuple to pure frames
|
||||
@@ -1147,49 +1146,43 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
"HF processor doesn't support video metadata. "
|
||||
"Timestamps will NOT be rendered. Please upgrade the model."
|
||||
)
|
||||
mm_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos]
|
||||
hf_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos]
|
||||
|
||||
processor_output = self.info.ctx.call_hf_processor(
|
||||
hf_processor,
|
||||
dict(
|
||||
text=[prompt_text], images=mm_data["images"], videos=mm_data["videos"]
|
||||
),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
return hf_inputs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "images" not in hf_data and "videos" not in hf_data:
|
||||
return processed_data
|
||||
|
||||
# Divide the processor_output into two modalities: image and video.
|
||||
if processor_output is not None:
|
||||
pixel_values = processor_output["images"]
|
||||
if pixel_values is not None:
|
||||
processor_output["images"] = self._pixel_values_norm(
|
||||
pixel_values, hf_processor_mm_kwargs
|
||||
)
|
||||
for key in list(processor_output.keys()):
|
||||
if processor_output[key] is None:
|
||||
del processor_output[key]
|
||||
continue
|
||||
if key == "grid_thw":
|
||||
grid_thw = processor_output["grid_thw"]
|
||||
pixel_values_all = processor_output["images"]
|
||||
# Identify elements where the first
|
||||
# dimension is greater than 1 and
|
||||
# treat them as the video modality
|
||||
mask = grid_thw[:, 0] > 1
|
||||
processor_output["video_grid_thw"] = grid_thw[mask]
|
||||
processor_output["image_grid_thw"] = grid_thw[~mask]
|
||||
image_patch_num = (
|
||||
processor_output["image_grid_thw"].prod(dim=1).sum()
|
||||
)
|
||||
processor_output["pixel_values"] = pixel_values_all[
|
||||
:image_patch_num
|
||||
]
|
||||
processor_output["pixel_values_videos"] = pixel_values_all[
|
||||
image_patch_num:
|
||||
]
|
||||
del processor_output["images"]
|
||||
pixel_values = processed_data["images"]
|
||||
if pixel_values is not None:
|
||||
processed_data["images"] = self._pixel_values_norm(pixel_values, hf_kwargs)
|
||||
for key in list(processed_data.keys()):
|
||||
if processed_data[key] is None:
|
||||
del processed_data[key]
|
||||
continue
|
||||
if key == "grid_thw":
|
||||
grid_thw = processed_data["grid_thw"]
|
||||
pixel_values_all = processed_data["images"]
|
||||
# Identify elements where the first
|
||||
# dimension is greater than 1 and
|
||||
# treat them as the video modality
|
||||
mask = grid_thw[:, 0] > 1
|
||||
processed_data["video_grid_thw"] = grid_thw[mask]
|
||||
processed_data["image_grid_thw"] = grid_thw[~mask]
|
||||
image_patch_num = processed_data["image_grid_thw"].prod(dim=1).sum()
|
||||
processed_data["pixel_values"] = pixel_values_all[:image_patch_num]
|
||||
processed_data["pixel_values_videos"] = pixel_values_all[
|
||||
image_patch_num:
|
||||
]
|
||||
del processed_data["images"]
|
||||
|
||||
processed_data = processor_output
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_prompt_updates(
|
||||
|
||||
@@ -38,6 +38,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.transformers_utils.processors.fireredasr2 import (
|
||||
FireRedASR2FeatureExtractor,
|
||||
@@ -228,37 +229,24 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing
|
||||
class FireRedASR2MultiModalProcessor(
|
||||
BaseMultiModalProcessor[FireRedASR2ProcessingInfo]
|
||||
):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
|
||||
return hf_inputs._replace(
|
||||
hf_kwargs=dict(
|
||||
hf_inputs.hf_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
)
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "labels" in processed_data:
|
||||
processed_data["input_ids"] = processed_data.pop("labels")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -49,7 +49,10 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
TimingContext,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
|
||||
from vllm.multimodal.processing.processor import (
|
||||
HFMultiModalInputs,
|
||||
MultiModalProcessingResult,
|
||||
)
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -752,42 +755,29 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
|
||||
|
||||
|
||||
class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
|
||||
return hf_inputs._replace(
|
||||
hf_kwargs=dict(
|
||||
hf_inputs.hf_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
)
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "labels" in processed_data:
|
||||
processed_data["input_ids"] = processed_data.pop("labels")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
) -> MultiModalProcessingResult:
|
||||
# Dithering injects noise into the extracted features, so the
|
||||
# feature extractor is not a pure function of its input. Since the
|
||||
# processing cache assumes that processor outputs are invariant
|
||||
|
||||
@@ -616,29 +616,29 @@ class FunAudioChatMultiModalProcessor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
|
||||
|
||||
audios = mm_data.get("audios", [])
|
||||
audios = mm_data.get("audio", [])
|
||||
if not audios:
|
||||
processed_data = BatchFeature({"input_ids": input_ids})
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
mm_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
assert isinstance(audios, Sequence)
|
||||
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
|
||||
sr = int(feature_extractor.sampling_rate)
|
||||
min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400)
|
||||
|
||||
@@ -690,8 +690,9 @@ class FunAudioChatMultiModalProcessor(
|
||||
}
|
||||
|
||||
processed_data = BatchFeature({"input_ids": input_ids, **mm_inputs})
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
mm_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -263,7 +263,7 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
|
||||
|
||||
|
||||
class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
|
||||
@@ -265,20 +265,9 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
|
||||
|
||||
|
||||
class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
if "audios" in mm_data:
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
|
||||
@@ -595,22 +595,25 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
|
||||
|
||||
|
||||
class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
if not hf_data:
|
||||
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
prompt_text = hf_data.pop("text")
|
||||
assert isinstance(prompt_text, str)
|
||||
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_kwargs)
|
||||
val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
|
||||
|
||||
if val is not None and val not in _SUPPORTED_SOFT_TOKENS:
|
||||
@@ -619,14 +622,12 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
f"Valid values are {_SUPPORTED_SOFT_TOKENS}."
|
||||
)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
|
||||
# ---- VIDEO HANDLING ----
|
||||
# Gemma4 decomposes video into timestamped image frames.
|
||||
# Each frame is processed with max_soft_tokens=70 through the
|
||||
# same vision tower, matching transformers processing_gemma4.py.
|
||||
video_outputs: dict[str, Any] = {}
|
||||
if videos := mm_data.pop("videos", []):
|
||||
if videos := hf_data.pop("videos", []):
|
||||
assert isinstance(videos, list)
|
||||
processor = self.info.get_hf_processor()
|
||||
|
||||
@@ -656,7 +657,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
timestamps = [idx / fps for idx in frame_indices]
|
||||
|
||||
# Process frames as images with max_soft_tokens=70
|
||||
video_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||
video_mm_kwargs = dict(hf_kwargs)
|
||||
# Override configured image options too: these inputs are video
|
||||
# frames, whose cache keys exclude images_kwargs.
|
||||
video_mm_kwargs["images_kwargs"] = {}
|
||||
@@ -726,18 +727,14 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
"video_timestamps": video_timestamps_per_video,
|
||||
}
|
||||
|
||||
# The processor accepts 'audio' not 'audios'.
|
||||
if "audios" in mm_data:
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
# Warn if any audio waveform exceeds the model's max duration.
|
||||
if "audio" in mm_data:
|
||||
if "audio" in hf_data:
|
||||
processor = self.info.get_hf_processor()
|
||||
sr = processor.feature_extractor.sampling_rate
|
||||
max_tokens = processor.audio_seq_length
|
||||
ms_per_tok = processor.audio_ms_per_token
|
||||
max_duration_s = max_tokens * ms_per_tok / 1000.0
|
||||
audios = mm_data["audio"]
|
||||
audios = hf_data["audio"]
|
||||
if not isinstance(audios, (list, tuple)):
|
||||
audios = [audios]
|
||||
for i, waveform in enumerate(audios):
|
||||
@@ -760,13 +757,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
# NOTE: This requires a corresponding type annotation on the
|
||||
# HF side (Gemma4ProcessorKwargs.images_kwargs) so that
|
||||
# _merge_kwargs routes max_soft_tokens into images_kwargs.
|
||||
patched_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||
patched_mm_kwargs = dict(hf_kwargs)
|
||||
if val is not None and is_top_level_max_soft_tokens:
|
||||
patched_mm_kwargs["max_soft_tokens"] = val
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**patched_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
dict(text=prompt_text, **hf_data),
|
||||
patched_mm_kwargs,
|
||||
)
|
||||
|
||||
@@ -797,9 +794,9 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
processed_data["input_features_mask"] = unpadded_masks
|
||||
|
||||
processed_data.update(video_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -1582,52 +1582,59 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
|
||||
return prepared_data, prepared_kwargs
|
||||
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
if not hf_data:
|
||||
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
prompt_text = hf_data.pop("text")
|
||||
assert isinstance(prompt_text, str)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
processor = self.info.get_hf_processor(**hf_kwargs)
|
||||
|
||||
use_direct_path = (
|
||||
not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
|
||||
)
|
||||
if use_direct_path:
|
||||
prepared_data, prepared_kwargs = self._get_direct_path_inputs(
|
||||
mm_data, hf_processor_mm_kwargs
|
||||
hf_data, hf_kwargs
|
||||
)
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**prepared_kwargs),
|
||||
dict(text=prompt_text, **prepared_data),
|
||||
prepared_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
videos = mm_data.get("videos")
|
||||
if isinstance(videos, list) and videos:
|
||||
if (
|
||||
"videos" in hf_data
|
||||
and isinstance(videos := hf_data["videos"], list)
|
||||
and len(videos) > 0
|
||||
):
|
||||
# Videos are processed item-wise below; remove them from hf_data so
|
||||
# the combined call at the end does not re-sample the raw frames.
|
||||
del hf_data["videos"]
|
||||
video_grid_thw_lst = []
|
||||
pixel_values_videos_lst = []
|
||||
frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor)
|
||||
swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id
|
||||
mm_data.pop("videos")
|
||||
for item in videos:
|
||||
video_array, metadata = item
|
||||
|
||||
# don't update mm_kwargs inplace
|
||||
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
|
||||
video_mm_kwargs = dict(**hf_kwargs)
|
||||
video_mm_kwargs["do_sample_frames"] = metadata.get(
|
||||
"do_sample_frames", True
|
||||
)
|
||||
@@ -1667,9 +1674,9 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
swap_video_frame_tokens = False
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
dict(text=prompt_text, **hf_data),
|
||||
hf_kwargs,
|
||||
)
|
||||
if swap_video_frame_tokens:
|
||||
input_ids = processed_data["input_ids"]
|
||||
@@ -1677,9 +1684,9 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
processed_data["input_ids"] = input_ids
|
||||
|
||||
processed_data.update(video_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -44,6 +44,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
@@ -739,26 +740,24 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
|
||||
chunk_counts.append(min(n_chunks, max_windows))
|
||||
return chunk_counts
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
if "audios" in mm_data:
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
|
||||
return hf_inputs._replace(
|
||||
hf_kwargs=dict(
|
||||
hf_inputs.hf_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
)
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
|
||||
@@ -110,7 +110,7 @@ class GraniteSpeechAudioInputs(TensorSchema):
|
||||
"""List of audio embedding sizes for each item in batch."""
|
||||
|
||||
|
||||
class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
|
||||
class GraniteSpeechProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
feature_extractor = self.get_hf_processor().audio_processor
|
||||
|
||||
@@ -135,7 +135,7 @@ class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
### Input Processing & Multimodal utils
|
||||
class GraniteSpeechMultiModalProcessor(
|
||||
BaseMultiModalProcessor[GraniteSpeechMultiModalProcessingInfo]
|
||||
BaseMultiModalProcessor[GraniteSpeechProcessingInfo]
|
||||
):
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
@@ -181,23 +181,9 @@ class GraniteSpeechMultiModalProcessor(
|
||||
)
|
||||
]
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
|
||||
if audios:
|
||||
# GraniteSpeechFeatureExtractor accepts "audio"
|
||||
mm_data["audio"] = audios
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
@@ -216,7 +202,7 @@ class GraniteSpeechMultiModalProcessor(
|
||||
|
||||
|
||||
class GraniteSpeechDummyInputsBuilder(
|
||||
BaseDummyInputsBuilder[GraniteSpeechMultiModalProcessingInfo]
|
||||
BaseDummyInputsBuilder[GraniteSpeechProcessingInfo]
|
||||
):
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
@@ -565,7 +551,7 @@ class GraniteSpeechCTCEncoder(nn.Module):
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
GraniteSpeechMultiModalProcessor,
|
||||
info=GraniteSpeechMultiModalProcessingInfo,
|
||||
info=GraniteSpeechProcessingInfo,
|
||||
dummy_inputs=GraniteSpeechDummyInputsBuilder,
|
||||
)
|
||||
class GraniteSpeechForConditionalGeneration(
|
||||
|
||||
@@ -12,8 +12,8 @@ from .granite_speech import (
|
||||
GraniteSpeechCTCEncoder,
|
||||
GraniteSpeechDummyInputsBuilder,
|
||||
GraniteSpeechForConditionalGeneration,
|
||||
GraniteSpeechMultiModalProcessingInfo,
|
||||
GraniteSpeechMultiModalProcessor,
|
||||
GraniteSpeechProcessingInfo,
|
||||
)
|
||||
|
||||
ISO639_1_SUPPORTED_LANGS = {
|
||||
@@ -60,7 +60,7 @@ class GraniteSpeechPlusCTCEncoder(GraniteSpeechCTCEncoder):
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
GraniteSpeechMultiModalProcessor,
|
||||
info=GraniteSpeechMultiModalProcessingInfo,
|
||||
info=GraniteSpeechProcessingInfo,
|
||||
dummy_inputs=GraniteSpeechDummyInputsBuilder,
|
||||
)
|
||||
class GraniteSpeechPlusForConditionalGeneration(GraniteSpeechForConditionalGeneration):
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.multimodal.parse import (
|
||||
MultiModalDataItems,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import (
|
||||
MultiModalProcessingInfo,
|
||||
MultiModalProcessingResult,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
TimingContext,
|
||||
@@ -139,7 +139,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
) -> MultiModalProcessingResult:
|
||||
# The processor logic is different for len(images) <= 1 vs > 1
|
||||
# Since the processing cache assumes that the processor output is
|
||||
# invariant of how many images are passed per prompt, we only
|
||||
|
||||
@@ -261,44 +261,8 @@ class HCXVisionV2MultiModalProcessor(
|
||||
):
|
||||
"""Multimodal processor for HyperCLOVAX V2 (32B Think model)."""
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
images = mm_data.get("images")
|
||||
videos = mm_data.get("videos")
|
||||
|
||||
# Get the HF processor
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
# Build data dict for HF processor (images/videos only)
|
||||
# NOTE: We pass the prompt as-is without token normalization.
|
||||
# Token expansion is handled by vLLM via _get_prompt_updates.
|
||||
data: dict[str, object] = dict(
|
||||
text=prompt_text,
|
||||
images=images,
|
||||
videos=videos,
|
||||
)
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
hf_processor=hf_processor,
|
||||
data=data,
|
||||
kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -52,6 +52,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -240,31 +241,32 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
|
||||
|
||||
|
||||
class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]):
|
||||
def _apply_hf_processor_main(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
image_processor = self.info.get_hf_processor().image_processor
|
||||
if getattr(image_processor, "backend", "pil") == "pil":
|
||||
hf_processor_mm_kwargs = {
|
||||
"input_data_format": "channels_last",
|
||||
**hf_processor_mm_kwargs,
|
||||
}
|
||||
hf_inputs = hf_inputs._replace(
|
||||
hf_kwargs={
|
||||
"input_data_format": "channels_last",
|
||||
**hf_inputs.hf_kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
return hf_inputs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
return processed_data
|
||||
|
||||
images = mm_data.get("images", [])
|
||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||
@@ -289,8 +291,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
|
||||
processed_data["pixel_values"].squeeze_(0)
|
||||
processed_data["pixel_attention_mask"].squeeze_(0)
|
||||
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -335,28 +335,30 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
|
||||
class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]):
|
||||
"""Basic image-only MultiModalProcessor for InternS1-style models."""
|
||||
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
if not hf_data:
|
||||
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
prompt_text = hf_data.pop("text")
|
||||
assert isinstance(prompt_text, str)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
videos = mm_data.pop("videos", [])
|
||||
images = mm_data.pop("images", [])
|
||||
videos = hf_data.pop("videos", [])
|
||||
images = hf_data.pop("images", [])
|
||||
assert isinstance(videos, list)
|
||||
assert isinstance(images, list)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_kwargs)
|
||||
tokenizer = hf_processor.tokenizer
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
@@ -374,9 +376,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
image_pixel_values = []
|
||||
for image in images:
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
dict(text=hf_processor.image_token, **{"images": image}),
|
||||
hf_processor_mm_kwargs,
|
||||
hf_kwargs,
|
||||
)
|
||||
image_pixel_values.append(processed_data.pop("pixel_values"))
|
||||
|
||||
@@ -398,9 +400,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
video_pixel_values = []
|
||||
for video in videos:
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
dict(text=hf_processor.video_token, **{"videos": video}),
|
||||
hf_processor_mm_kwargs,
|
||||
hf_kwargs,
|
||||
)
|
||||
video_pixel_values.append(processed_data.pop("pixel_values"))
|
||||
|
||||
@@ -430,9 +432,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
processed_data.update(text_outputs)
|
||||
processed_data.update(image_outputs)
|
||||
processed_data.update(video_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.model_executor.layers.pooler import DispatchPooler
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItems
|
||||
from vllm.multimodal.processing.processor import (
|
||||
MultiModalProcessingInfo,
|
||||
MultiModalProcessingResult,
|
||||
ProcessorInputs,
|
||||
TimingContext,
|
||||
)
|
||||
@@ -63,19 +63,19 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
mm_info = super()._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
) -> MultiModalProcessingResult:
|
||||
mm_res = super()._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
# Score inputs are query-first, while the prompt template is document-first.
|
||||
mm_kwargs = MultiModalKwargsItems(
|
||||
{
|
||||
modality: list(reversed(items))
|
||||
for modality, items in mm_info.kwargs.items()
|
||||
for modality, items in mm_res.kwargs.items()
|
||||
}
|
||||
)
|
||||
mm_hashes = {
|
||||
modality: list(reversed(hashes))
|
||||
for modality, hashes in mm_info.hashes.items()
|
||||
for modality, hashes in mm_res.hashes.items()
|
||||
}
|
||||
mm_prompt_updates = {
|
||||
modality: [
|
||||
@@ -85,10 +85,11 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
]
|
||||
for item_idx, updates in enumerate(reversed(item_updates))
|
||||
]
|
||||
for modality, item_updates in mm_info.prompt_updates.items()
|
||||
for modality, item_updates in mm_res.prompt_updates.items()
|
||||
}
|
||||
|
||||
return MultiModalProcessingInfo(
|
||||
return MultiModalProcessingResult(
|
||||
prompt_ids=mm_res.prompt_ids,
|
||||
kwargs=mm_kwargs,
|
||||
hashes=mm_hashes,
|
||||
prompt_updates=mm_prompt_updates,
|
||||
|
||||
@@ -459,24 +459,27 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
||||
def media_token_id(self) -> int:
|
||||
return self.info.get_hf_config().text_config.eos_token_id + 1
|
||||
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Run the underlying HF processor on text and image data."""
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data or not mm_data.get("images", []):
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
if not hf_data or not hf_data.get("images", []):
|
||||
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
prompt_text = hf_data.pop("text")
|
||||
assert isinstance(prompt_text, str)
|
||||
|
||||
# Images
|
||||
image_inputs = mm_data.get("images", [])
|
||||
image_inputs = hf_data.get("images", [])
|
||||
pixel_sizes = []
|
||||
if not isinstance(image_inputs[0], Image.Image):
|
||||
image_inputs = [Image.fromarray(image) for image in image_inputs]
|
||||
@@ -530,8 +533,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
||||
pixel_sizes=torch.tensor(pixel_sizes),
|
||||
)
|
||||
processed_data = BatchFeature(combined_outputs, tensor_type="pt")
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -1160,7 +1160,7 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):
|
||||
|
||||
|
||||
class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_prompt_updates(
|
||||
|
||||
@@ -373,7 +373,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
|
||||
|
||||
|
||||
class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_prompt_updates(
|
||||
|
||||
@@ -47,6 +47,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
HFMultiModalInputs,
|
||||
ProcessorInputs,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -227,27 +228,17 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser):
|
||||
class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]):
|
||||
"""vLLM multi-modal processor wrapper for Kimi-Audio."""
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Call the HuggingFace processor."""
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
# Convert mm_data format: {'audios': [...]} -> {'audio': ...}
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
# Convert audio format: [(array, sr), ...] -> [array, ...]
|
||||
# KimiAudioProcessor expects raw numpy arrays
|
||||
if audios:
|
||||
mm_data = hf_inputs.hf_data
|
||||
if audios := mm_data.pop("audio", []):
|
||||
audio_arrays = []
|
||||
for aud in audios:
|
||||
if isinstance(aud, (tuple, list)) and len(aud) == 2:
|
||||
@@ -259,14 +250,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
|
||||
audio_arrays.append(aud)
|
||||
mm_data["audio"] = audio_arrays
|
||||
|
||||
# Use the context's call_hf_processor for proper handling
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return hf_inputs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -236,7 +236,7 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
|
||||
|
||||
|
||||
class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -406,7 +406,7 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):
|
||||
|
||||
|
||||
class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import TypeVar
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -35,8 +34,6 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
|
||||
_I = TypeVar("_I", bound=Mistral3ProcessingInfo)
|
||||
|
||||
|
||||
class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
|
||||
def get_vision_encoder_info(
|
||||
@@ -49,8 +46,8 @@ class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
|
||||
return PixtralHFEncoderInfo(self.get_hf_config())
|
||||
|
||||
|
||||
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessingInfo]):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
@@ -62,27 +59,6 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
|
||||
if not mm_data:
|
||||
return processed_data
|
||||
|
||||
input_ids = processed_data.get("input_ids")
|
||||
if input_ids is not None:
|
||||
processor = self.info.get_hf_processor()
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
break_id = vocab.get(processor.image_break_token)
|
||||
end_id = vocab.get(processor.image_end_token)
|
||||
|
||||
# create mask to remove break/end tokens
|
||||
keep_mask = ~torch.isin(
|
||||
input_ids,
|
||||
torch.tensor([break_id, end_id]),
|
||||
)
|
||||
|
||||
processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0)
|
||||
if "attention_mask" in processed_data:
|
||||
processed_data["attention_mask"] = processed_data["attention_mask"][
|
||||
keep_mask
|
||||
].unsqueeze(0)
|
||||
|
||||
# un-pad pixel_values per-image so caches remain independent.
|
||||
pixel_values = processed_data.get("pixel_values")
|
||||
if pixel_values is not None:
|
||||
|
||||
@@ -320,7 +320,7 @@ class PixtralHFProcessingInfo(BaseLlavaProcessingInfo):
|
||||
|
||||
|
||||
class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
|
||||
@@ -209,7 +209,7 @@ class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]):
|
||||
|
||||
|
||||
class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -324,33 +324,36 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
pixel_values_videos=MultiModalFieldConfig.batched("video"),
|
||||
)
|
||||
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
if not hf_data:
|
||||
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
prompt_text = hf_data.pop("text")
|
||||
assert isinstance(prompt_text, str)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
videos = mm_data.pop("videos", [])
|
||||
videos = hf_data.pop("videos", [])
|
||||
assert isinstance(videos, list)
|
||||
|
||||
if not videos:
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
dict(text=prompt_text, **hf_data),
|
||||
hf_kwargs,
|
||||
)
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
# LLaVA-OneVision processor doesn't support multiple videos
|
||||
# with different sizes when converting back to tensors
|
||||
@@ -360,13 +363,13 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
image_token = processor.image_token
|
||||
video_token = processor.video_token
|
||||
|
||||
images = mm_data.pop("images", [])
|
||||
images = hf_data.pop("images", [])
|
||||
assert isinstance(images, list)
|
||||
if images:
|
||||
processor_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
dict(text=image_token * len(images), **{"images": images}),
|
||||
hf_processor_mm_kwargs,
|
||||
hf_kwargs,
|
||||
)
|
||||
image_outputs = {
|
||||
k: v
|
||||
@@ -379,9 +382,9 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
pixel_values_videos = []
|
||||
for video in videos:
|
||||
item_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
dict(text=video_token, **{"videos": video}),
|
||||
hf_processor_mm_kwargs,
|
||||
hf_kwargs,
|
||||
)
|
||||
|
||||
pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
|
||||
@@ -394,8 +397,9 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
**video_outputs,
|
||||
)
|
||||
processed_data = BatchFeature(combined_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -1556,23 +1556,26 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
|
||||
class LlavaOnevision2MultiModalProcessor(
|
||||
BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]
|
||||
):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
if not hf_data:
|
||||
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
prompt_text = hf_data.pop("text")
|
||||
assert isinstance(prompt_text, str)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_kwargs)
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_kwargs)
|
||||
merged_kwargs.setdefault("return_tensors", "pt")
|
||||
call_kwargs = {
|
||||
k: v
|
||||
@@ -1589,17 +1592,17 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
"codec_config",
|
||||
}
|
||||
}
|
||||
mm_data = dict(mm_data)
|
||||
|
||||
# Explicit None + length checks: ``mm_data[...]`` may be a list, numpy
|
||||
# array, or tensor, and ``and <array>`` would raise on the ambiguous
|
||||
# truth value of a multi-element array.
|
||||
_videos = mm_data.get("videos")
|
||||
_videos = hf_data.get("videos")
|
||||
if _videos is not None and not isinstance(_videos, Sized):
|
||||
raise TypeError(f"object of type '{type(_videos).__name__}' has no len()")
|
||||
videos_present = _videos is not None and len(_videos) > 0
|
||||
|
||||
codec_video_paths = (
|
||||
_extract_codec_video_paths(mm_data["videos"]) if videos_present else None
|
||||
_extract_codec_video_paths(hf_data["videos"]) if videos_present else None
|
||||
)
|
||||
is_codec_marker = codec_video_paths is not None
|
||||
# Fallback: caller passed video_backend=codec via mm_processor_kwargs
|
||||
@@ -1613,7 +1616,7 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
and call_kwargs.get("video_backend") == "codec"
|
||||
)
|
||||
if is_codec_kwarg:
|
||||
raw = mm_data["videos"]
|
||||
raw = hf_data["videos"]
|
||||
if isinstance(raw, str):
|
||||
codec_video_paths = [raw]
|
||||
elif isinstance(raw, (list, tuple)) and all(
|
||||
@@ -1638,7 +1641,7 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
# Codec backend: HF processor consumes the path string directly and
|
||||
# performs decode + canvas-packing internally. The dummy ndarray
|
||||
# we attached during prepare_codec_video_input is discarded here.
|
||||
mm_data["videos"] = (
|
||||
hf_data["videos"] = (
|
||||
codec_video_paths
|
||||
if len(codec_video_paths) > 1
|
||||
else codec_video_paths[0]
|
||||
@@ -1648,11 +1651,9 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
# ``video_backend="codec"`` via mm_kwargs so the wrapped processor
|
||||
# dispatches to its codec branch.
|
||||
output = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(
|
||||
**{**hf_processor_mm_kwargs, "video_backend": "codec"}
|
||||
),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
{**hf_processor_mm_kwargs, "video_backend": "codec"},
|
||||
self.info.get_hf_processor(**{**hf_kwargs, "video_backend": "codec"}),
|
||||
dict(text=prompt_text, **hf_data),
|
||||
{**hf_kwargs, "video_backend": "codec"},
|
||||
)
|
||||
data = dict(output)
|
||||
processed_data = BatchFeature(
|
||||
@@ -1660,8 +1661,9 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
data, codec_video_paths, hf_processor
|
||||
)
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
# ---- Frame backend (registered LlavaOnevision2VideoBackend) ------
|
||||
# Every non-codec video reaches here as a ``(frames_ndarray, metadata)``
|
||||
@@ -1676,7 +1678,7 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
# provided by the backend's ``compute_frames_index_to_sample``; SSRF /
|
||||
# local-file gating is enforced by the connector before decoding.
|
||||
if videos_present:
|
||||
timestamp_decimals = hf_processor_mm_kwargs.get(
|
||||
timestamp_decimals = hf_kwargs.get(
|
||||
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
|
||||
)
|
||||
if not isinstance(
|
||||
@@ -1690,7 +1692,7 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
|
||||
per_video_frames: list[list[Image.Image]] = []
|
||||
per_video_timestamps: list[list[float]] = []
|
||||
videos = mm_data["videos"]
|
||||
videos = hf_data["videos"]
|
||||
if not isinstance(videos, Iterable):
|
||||
raise TypeError(f"Unsupported videos input: {type(videos)}")
|
||||
for item in videos:
|
||||
@@ -1714,7 +1716,7 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
# order) -- otherwise mixed image+video requests bind frames to the
|
||||
# wrong placeholder. ``row_is_video`` labels each grid row so outputs
|
||||
# can be split back into per-modality keys below.
|
||||
merged_mm_data = dict(mm_data)
|
||||
merged_mm_data = dict(hf_data)
|
||||
existing_images = merged_mm_data.pop("images", None)
|
||||
caller_images: list[Image.Image] = []
|
||||
if existing_images:
|
||||
@@ -1760,9 +1762,9 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
# **kwargs to the image processor, so passing the full merged kwarg
|
||||
# set here is a no-op beyond return_tensors/padding.
|
||||
output = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
dict(text=new_prompt, **merged_mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
hf_kwargs,
|
||||
)
|
||||
data = dict(output)
|
||||
|
||||
@@ -1820,20 +1822,22 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
)
|
||||
|
||||
processed_data = BatchFeature(data)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
# ---- Image-only / text-only call --------------------------------
|
||||
# No videos present: delegate to ``ctx.call_hf_processor``, which
|
||||
# runs the wrapped processor over the (possibly empty) image set and
|
||||
# applies float-tensor dtype postprocessing automatically.
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
dict(text=prompt_text, **hf_data),
|
||||
hf_kwargs,
|
||||
)
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _rename_codec_outputs_to_video(
|
||||
self,
|
||||
|
||||
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.midashenglm import DashengConfig
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -582,35 +583,33 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
|
||||
class MiDashengLMMultiModalProcessor(
|
||||
BaseMultiModalProcessor[MiDashengLMProcessingInfo]
|
||||
):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
assert isinstance(audios, list)
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
min_audio_len = self.info.get_min_audio_len()
|
||||
processed_audios = [
|
||||
np.pad(
|
||||
audio,
|
||||
(0, min_audio_len - audio.shape[-1]),
|
||||
mode="constant",
|
||||
constant_values=0,
|
||||
)
|
||||
if isinstance(audio, np.ndarray) and audio.shape[-1] < min_audio_len
|
||||
else audio
|
||||
for audio in audios
|
||||
]
|
||||
if audios := hf_inputs.hf_data.get("audio"):
|
||||
assert isinstance(audios, list)
|
||||
|
||||
if processed_audios:
|
||||
mm_data["audio"] = processed_audios
|
||||
min_audio_len = self.info.get_min_audio_len()
|
||||
hf_inputs.hf_data["audio"] = [
|
||||
np.pad(
|
||||
audio,
|
||||
(0, min_audio_len - audio.shape[-1]),
|
||||
mode="constant",
|
||||
constant_values=0,
|
||||
)
|
||||
if isinstance(audio, np.ndarray) and audio.shape[-1] < min_audio_len
|
||||
else audio
|
||||
for audio in audios
|
||||
]
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
return hf_inputs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -46,6 +46,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.transformers_utils.configs.mimo_v2_omni import Mimo_VLVisionConfig
|
||||
from vllm.transformers_utils.processors.mimo_v2_omni import (
|
||||
MiMoOmniProcessor,
|
||||
@@ -930,27 +931,14 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
||||
fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")
|
||||
return fields
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.
|
||||
Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses
|
||||
the singular form.
|
||||
"""
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
# Remap audios → audio (MiMoOmniProcessor uses singular param name)
|
||||
if "audios" in mm_data:
|
||||
mm_data = {**mm_data, "audio": mm_data["audios"]}
|
||||
mm_data = {k: v for k, v in mm_data.items() if k != "audios"}
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo."""
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
mm_data = hf_inputs.hf_data
|
||||
|
||||
# Handle video_audio items: convert video part to (TCHW, timestamps) tuple
|
||||
if "video_audio" in mm_data:
|
||||
@@ -992,7 +980,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
||||
audio=va_item.audio,
|
||||
)
|
||||
)
|
||||
mm_data = {**mm_data, "video_audio": va_converted}
|
||||
mm_data["video_audio"] = va_converted
|
||||
|
||||
if "videos" in mm_data:
|
||||
converted: list[tuple[torch.Tensor, torch.Tensor]] = []
|
||||
@@ -1026,15 +1014,9 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
||||
timestamps = torch.arange(T, dtype=torch.float32) / self._INPUT_FPS
|
||||
converted.append((frames, timestamps))
|
||||
|
||||
mm_data = {**mm_data, "videos": converted}
|
||||
mm_data["videos"] = converted
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return hf_inputs
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -445,7 +445,7 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, NestedTensors]:
|
||||
if (audios := mm_data.get("audios")) is None:
|
||||
if (audios := mm_data.get("audio")) is None:
|
||||
return {}
|
||||
|
||||
mm_items = self.info.parse_mm_data({"audio": audios}, validate=False)
|
||||
|
||||
@@ -1059,19 +1059,18 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
|
||||
mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
|
||||
mm_inputs = self.process_mm_inputs(mm_data, hf_kwargs)
|
||||
|
||||
processed_data = BatchFeature(
|
||||
{
|
||||
@@ -1079,8 +1078,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
**mm_inputs,
|
||||
}
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
mm_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -248,7 +248,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
|
||||
return ImageSize(width=width, height=height)
|
||||
|
||||
|
||||
class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo]):
|
||||
class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
|
||||
@@ -280,7 +280,7 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo])
|
||||
|
||||
|
||||
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
|
||||
@@ -589,7 +589,7 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
|
||||
class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
|
||||
@@ -1122,6 +1122,9 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
|
||||
|
||||
|
||||
class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||
processor = self.info.get_hf_processor()
|
||||
|
||||
@@ -1140,28 +1143,28 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
||||
dict(tokens=tokens),
|
||||
)["input_ids"].tolist()
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
|
||||
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not processor_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
hf_processor = self.info.get_hf_processor(**hf_kwargs)
|
||||
return self.info.ctx.call_hf_processor(
|
||||
hf_processor.process,
|
||||
dict(text=prompt_text, **processor_data),
|
||||
hf_processor_mm_kwargs,
|
||||
hf_data,
|
||||
hf_kwargs,
|
||||
)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not hf_data:
|
||||
return processed_data
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_kwargs)
|
||||
tokenizer = hf_processor.tokenizer
|
||||
image_patch_id = tokenizer.vocab[IMAGE_PATCH_TOKEN]
|
||||
|
||||
@@ -1169,7 +1172,7 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
||||
|
||||
processed_data.pop("input_ids")
|
||||
|
||||
if (images := processor_data.get("images")) is not None:
|
||||
if (images := hf_data.get("images")) is not None:
|
||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = [
|
||||
@@ -1193,8 +1196,6 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
||||
processed_data["num_crops"] = num_crops
|
||||
processed_data["img_patch_id"] = image_patch_id
|
||||
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -1932,6 +1932,9 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
|
||||
|
||||
|
||||
class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||
processor = self.info.get_hf_processor()
|
||||
tokenizer = processor.tokenizer
|
||||
@@ -1946,19 +1949,20 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
|
||||
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
if not hf_data:
|
||||
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
mm_data = dict(processor_data)
|
||||
prompt_text = hf_data.pop("text")
|
||||
assert isinstance(prompt_text, str)
|
||||
|
||||
hf_config = self.info.get_hf_config()
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_kwargs)
|
||||
|
||||
def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature:
|
||||
res = hf_processor(text=text, images=images, videos=videos, **kwargs)
|
||||
@@ -1973,7 +1977,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
tokenizer = hf_processor.tokenizer
|
||||
image_processor = hf_processor.image_processor
|
||||
|
||||
if videos := mm_data.pop("videos", []):
|
||||
if videos := hf_data.pop("videos", []):
|
||||
assert isinstance(videos, Sequence)
|
||||
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
|
||||
|
||||
@@ -1993,16 +1997,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
# NOTE: metadata.frames_indices indicates
|
||||
# the sampled frames indices of pre-sampled videos, which is
|
||||
# used to calculate the timestamps. Make sure that
|
||||
# do_sample_frames in hf_processor_mm_kwargs is false for
|
||||
# do_sample_frames in hf_kwargs is false for
|
||||
# presampled videos.
|
||||
|
||||
# NOTE: a copy of hf_processor_mm_kwargs is created to update
|
||||
# NOTE: a copy of hf_kwargs is created to update
|
||||
# do_sample_frames, otherwise mm_hash for the object will be
|
||||
# incorrect.
|
||||
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
|
||||
video_mm_kwargs = dict(**hf_kwargs)
|
||||
if "do_sample_frames" not in video_mm_kwargs:
|
||||
# molmo_utils already has "do_sample_frames" in
|
||||
# hf_processor_mm_kwargs, don't overwrite it.
|
||||
# hf_kwargs, don't overwrite it.
|
||||
video_mm_kwargs["do_sample_frames"] = metadata.get(
|
||||
"do_sample_frames", False
|
||||
)
|
||||
@@ -2067,11 +2071,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
patched_call,
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
dict(text=prompt_text, **hf_data),
|
||||
hf_kwargs,
|
||||
)
|
||||
|
||||
if (images := mm_data.get("images")) is not None:
|
||||
if (images := hf_data.get("images")) is not None:
|
||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = [
|
||||
@@ -2113,10 +2117,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
)
|
||||
|
||||
processed_data.update(all_video_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
processed_data.pop("input_ids")
|
||||
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -1356,32 +1356,17 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo
|
||||
|
||||
|
||||
class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]):
|
||||
def _apply_hf_processor_main(
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
if audios:
|
||||
mm_data["audio"] = audios
|
||||
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||
processor_kwargs = _filter_moss_audio_processor_config(hf_processor_mm_kwargs)
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
processor_kwargs = _filter_moss_audio_processor_config(dict(hf_kwargs))
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**processor_kwargs),
|
||||
mm_data,
|
||||
hf_data,
|
||||
{},
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -66,7 +66,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import ProcessorInputs
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs, ProcessorInputs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -465,17 +465,19 @@ class MossTranscribeDiarizeDummyInputsBuilder(
|
||||
class MossTranscribeDiarizeMultiModalProcessor(
|
||||
BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]
|
||||
):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
audios = _get_audios_from_mm_data(mm_data)
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
if hf_inputs.hf_data:
|
||||
hf_inputs.hf_data["audio"] = _get_audios_from_mm_data(hf_inputs.hf_data)
|
||||
|
||||
return dict(audio=audios), hf_processor_mm_kwargs
|
||||
return hf_inputs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
|
||||
@@ -275,16 +275,18 @@ class MuseGlimmerMultiModalProcessor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
processor = self.info.get_hf_processor(**hf_kwargs)
|
||||
tokenizer = processor.tokenizer
|
||||
config = self.info.get_hf_config()
|
||||
images = mm_data.get("images", ())
|
||||
@@ -371,8 +373,9 @@ class MuseGlimmerMultiModalProcessor(
|
||||
video_feature_sizes=torch.tensor(video_sizes),
|
||||
)
|
||||
processed_data = BatchFeature(data=data, tensor_type=None)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
mm_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -70,6 +70,7 @@ from vllm.multimodal.processing import (
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
HFMultiModalInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
@@ -196,6 +197,9 @@ NanoNemotronVLVideoInputs: TypeAlias = (
|
||||
|
||||
class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor:
|
||||
# Not accepted by NanoNemotronVLProcessor.__init__
|
||||
kwargs.pop("truncation", None)
|
||||
|
||||
return self.ctx.init_processor(
|
||||
NanoNemotronVLProcessor,
|
||||
config=self.get_hf_config(),
|
||||
@@ -358,9 +362,23 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
|
||||
class NanoNemotronVLMultiModalProcessor(
|
||||
BaseMultiModalProcessor[NanoNemotronVLProcessingInfo]
|
||||
):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
# NanoNemotronVLProcessor expects "audios" instead of "audio"
|
||||
hf_data = hf_inputs.hf_data
|
||||
if "audio" in hf_data:
|
||||
hf_data["audios"] = hf_data.pop("audio")
|
||||
|
||||
return hf_inputs
|
||||
|
||||
def _get_image_fields_config(self, hf_inputs: BatchFeature):
|
||||
if self.info.is_dynamic_tiler:
|
||||
pixel_values_flat = MultiModalFieldConfig.batched("image")
|
||||
@@ -730,15 +748,12 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
# Bypass the cached path: the HF processor must receive the
|
||||
# prompt (with injected <so_embedding>) and the audio data
|
||||
# together so it can perform audio-token replacement natively.
|
||||
mm_info = self._apply_hf_processor(inputs, timing_ctx)
|
||||
prompt_ids = self._postprocess_prompt(inputs.prompt)
|
||||
mm_res = self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
with timing_ctx.record("apply_prompt_updates"):
|
||||
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
|
||||
mm_items=mm_items,
|
||||
prompt_ids=prompt_ids,
|
||||
mm_kwargs=mm_info.kwargs,
|
||||
mm_prompt_updates=mm_info.prompt_updates,
|
||||
mm_res=mm_res,
|
||||
)
|
||||
|
||||
mm_placeholder_ranges = {
|
||||
@@ -749,8 +764,8 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
return MultiModalInput(
|
||||
type="multimodal",
|
||||
prompt_token_ids=prompt_ids,
|
||||
mm_kwargs=mm_info.kwargs,
|
||||
mm_hashes=mm_info.hashes,
|
||||
mm_kwargs=mm_res.kwargs,
|
||||
mm_hashes=mm_res.hashes,
|
||||
mm_placeholders=mm_placeholder_ranges,
|
||||
)
|
||||
|
||||
|
||||
@@ -326,6 +326,9 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]):
|
||||
|
||||
|
||||
class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def image_indicators_to_visual_tokens(
|
||||
self,
|
||||
image_indicators: list[int],
|
||||
@@ -340,9 +343,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
|
||||
# -300 is image_atom token, filter them out
|
||||
return [vte_vocab_size + x + 300 for x in image_indicators if x < -300]
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
|
||||
@@ -327,7 +327,7 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
|
||||
if x >= INDICATOR_IDS[0]
|
||||
]
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
|
||||
@@ -243,36 +243,37 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing
|
||||
class PaddleOCRVLMultiModalProcessor(
|
||||
BaseMultiModalProcessor[PaddleOCRVLProcessingInfo]
|
||||
):
|
||||
def _apply_hf_processor_main(
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
final_mm_kwargs = dict(hf_processor_mm_kwargs or {})
|
||||
final_mm_kwargs = dict(hf_kwargs or {})
|
||||
final_mm_kwargs.setdefault("images_kwargs", {})
|
||||
# vLLM use PIL.Image, always set channel_last
|
||||
final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**final_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
hf_data,
|
||||
hf_kwargs,
|
||||
)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not hf_data:
|
||||
return processed_data
|
||||
|
||||
num_patches_per_image = processed_data["image_grid_thw"].prod(-1)
|
||||
processed_data["pixel_values"] = processed_data["pixel_values"].split(
|
||||
num_patches_per_image.tolist()
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -390,28 +390,9 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
|
||||
|
||||
|
||||
class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
return processed_data
|
||||
|
||||
input_ids = processed_data["input_ids"]
|
||||
assert isinstance(input_ids, torch.Tensor)
|
||||
|
||||
# Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
|
||||
# which will cause OverflowError when decoding the prompt_ids.
|
||||
# Therefore, we need to do an early replacement here
|
||||
input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -45,6 +45,7 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
HFMultiModalInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
ResolvedPromptUpdate,
|
||||
@@ -867,40 +868,38 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
|
||||
|
||||
|
||||
class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
def _apply_hf_processor_main(
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
# The Phi4MM processor expects "audios" instead of "audio"
|
||||
hf_data = hf_inputs.hf_data
|
||||
if "audio" in hf_data:
|
||||
hf_data["audios"] = hf_data.pop("audio")
|
||||
|
||||
sr = self.info.get_feature_extractor(**hf_inputs.hf_kwargs).sampling_rate
|
||||
if audio_data := hf_data.get("audios", []):
|
||||
assert isinstance(audio_data, Sequence)
|
||||
hf_data["audios"] = [(data, sr) for data in audio_data]
|
||||
|
||||
return hf_inputs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
if not hf_data:
|
||||
return processed_data
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
sr = self.info.get_feature_extractor(**hf_processor_mm_kwargs).sampling_rate
|
||||
raw_audio_data = mm_data.get("audios", [])
|
||||
assert isinstance(raw_audio_data, Sequence)
|
||||
audio_data = []
|
||||
for data in raw_audio_data:
|
||||
assert isinstance(data, (list, np.ndarray, torch.Tensor))
|
||||
audio_data.append(data)
|
||||
|
||||
processor_data = dict(text=prompt_text, **mm_data)
|
||||
if audio_data:
|
||||
processor_data["audios"] = [(data, sr) for data in audio_data]
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_kwargs)
|
||||
num_img_tokens = [
|
||||
self.info.get_num_image_tokens(
|
||||
image_width=img_size[0],
|
||||
@@ -911,21 +910,26 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
]
|
||||
processed_data["num_img_tokens"] = num_img_tokens
|
||||
|
||||
audio_features = processed_data["input_audio_embeds"]
|
||||
audios = hf_data.get("audios", [])
|
||||
assert isinstance(audios, list)
|
||||
audio_features = processed_data.get("input_audio_embeds")
|
||||
if audio_features is None:
|
||||
return processed_data
|
||||
|
||||
sr = self.info.get_feature_extractor(**hf_kwargs).sampling_rate
|
||||
feature_sizes = [
|
||||
self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data
|
||||
self.info.get_audio_num_frames(len(audio), sr) for audio, _ in audios
|
||||
]
|
||||
processed_data["input_audio_embeds"] = [
|
||||
audio_features[idx, :size] for idx, size in enumerate(feature_sizes)
|
||||
]
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
return dict(
|
||||
input_image_embeds=MultiModalFieldConfig.batched("image"),
|
||||
|
||||
@@ -143,39 +143,8 @@ class Phi4SiglipDummyInputsBuilder(
|
||||
class Phi4SiglipMultiModalProcessor(
|
||||
BaseMultiModalProcessor[Phi4SiglipProcessingInfo],
|
||||
):
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
processed = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# The HF processor's tokenizer_image_token() replaces the "<image>"
|
||||
# string with IMAGE_TOKEN_INDEX (-200) in input_ids. This breaks
|
||||
# vLLM's prompt-replacement pipeline which needs to find "<image>"
|
||||
# as normal sub-tokens. Re-tokenize with the plain tokenizer so
|
||||
# that "<image>" stays as sub-tokens and can be located by
|
||||
# PromptReplacement.
|
||||
# NOTE: tokenizer.__call__() (not .encode()) must be used so that
|
||||
# added/special tokens like <|user|>, <|end|> are kept as single IDs.
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
new_ids = tokenizer(prompt_text).input_ids
|
||||
processed["input_ids"] = torch.tensor([new_ids])
|
||||
|
||||
processed_data = processed
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -14,7 +14,7 @@ from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
|
||||
from mistral_common.protocol.instruct.messages import UserMessage
|
||||
from mistral_common.protocol.instruct.request import ChatCompletionRequest
|
||||
from packaging.version import Version
|
||||
from transformers import BatchFeature, PixtralVisionConfig
|
||||
from transformers import PixtralVisionConfig
|
||||
from transformers.models.pixtral.image_processing_pixtral import (
|
||||
_num_image_tokens as _get_pixtral_hf_num_image_tokens,
|
||||
)
|
||||
@@ -41,7 +41,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsOptionalItems,
|
||||
NestedTensors,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
@@ -53,7 +52,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
MultiModalPromptUpdates,
|
||||
HFMultiModalInputs,
|
||||
MultiModalProcessingResult,
|
||||
PlaceholderFeaturesInfo,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
@@ -67,7 +67,6 @@ from vllm.transformers_utils.processors.pixtral import (
|
||||
MistralCommonImageProcessor,
|
||||
MistralCommonPixtralProcessor,
|
||||
)
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
@@ -268,21 +267,19 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
prompt_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsOptionalItems,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
mm_res: MultiModalProcessingResult,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
mm_item_counts = mm_items.get_all_counts()
|
||||
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
|
||||
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
|
||||
self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
|
||||
self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
|
||||
|
||||
mm_placeholders = self._find_mm_placeholders(
|
||||
prompt_ids,
|
||||
mm_prompt_updates,
|
||||
mm_res.prompt_ids,
|
||||
mm_res.prompt_updates,
|
||||
)
|
||||
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
|
||||
|
||||
return prompt_ids, mm_placeholders
|
||||
return mm_res.prompt_ids, mm_placeholders
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
@@ -291,35 +288,17 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
return dict(images=MultiModalFieldConfig.batched("image"))
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
# Avoid padding issue
|
||||
return hf_inputs._replace(
|
||||
hf_kwargs=dict(hf_inputs.hf_kwargs, return_tensors=None)
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
# Avoid padding issue
|
||||
dict(**hf_processor_mm_kwargs, return_tensors=None),
|
||||
)
|
||||
|
||||
# Missing batch dimension
|
||||
if is_list_of(outputs["input_ids"], int):
|
||||
outputs["input_ids"] = [outputs["input_ids"]]
|
||||
|
||||
processed_data = outputs
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -82,6 +82,8 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
HFMultiModalInputs,
|
||||
MultiModalProcessingResult,
|
||||
MultiModalPromptUpdates,
|
||||
PlaceholderFeaturesInfo,
|
||||
PromptReplacement,
|
||||
@@ -498,6 +500,64 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
|
||||
class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
|
||||
):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
if hf_kwargs.get("use_audio_in_video"):
|
||||
num_videos = mm_counts.get("video", 0)
|
||||
if mm_counts.get("audio", 0) < num_videos:
|
||||
raise ValueError(
|
||||
"Video doesn't have audio track with `audio_in_video=True`"
|
||||
)
|
||||
|
||||
if num_videos and "text" in hf_inputs.hf_data:
|
||||
standalone_mm_counts = dict(mm_counts)
|
||||
standalone_mm_counts["audio"] -= num_videos
|
||||
hf_inputs.hf_data["text"] = self._get_hf_mm_text(standalone_mm_counts)
|
||||
|
||||
hf_data = hf_inputs.hf_data
|
||||
normalized_kwargs = hf_inputs.hf_kwargs
|
||||
audios = hf_data.pop("audio", [])
|
||||
|
||||
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
|
||||
if audios:
|
||||
hf_data["audio"] = audios
|
||||
elif (
|
||||
normalized_kwargs.get("use_audio_in_video")
|
||||
and "audio" not in hf_data
|
||||
and hf_data.get("videos")
|
||||
):
|
||||
raise ValueError(
|
||||
"Video doesn't have audio track with `audio_in_video=True`"
|
||||
)
|
||||
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(normalized_kwargs)
|
||||
if hf_data.get("videos") and (
|
||||
merged.keys() & {"size", "min_pixels", "max_pixels"}
|
||||
):
|
||||
normalized_kwargs = dict(normalized_kwargs)
|
||||
video_size = dict(self.info.get_hf_processor().video_processor.size)
|
||||
size_override = merged.get("size")
|
||||
if size_override is not None:
|
||||
video_size = video_size | size_override
|
||||
min_pixels = merged.get("min_pixels")
|
||||
if min_pixels is not None:
|
||||
video_size["shortest_edge"] = min_pixels
|
||||
max_pixels = merged.get("max_pixels")
|
||||
if max_pixels is not None:
|
||||
video_size["longest_edge"] = max_pixels
|
||||
normalized_kwargs["size"] = video_size
|
||||
|
||||
return hf_inputs._replace(hf_kwargs=normalized_kwargs)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
@@ -570,12 +630,14 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
prompt_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsItems,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
mm_res: MultiModalProcessingResult,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
"""Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`."""
|
||||
mm_item_counts = mm_items.get_all_counts()
|
||||
prompt_ids = mm_res.prompt_ids
|
||||
mm_kwargs = mm_res.kwargs
|
||||
mm_prompt_updates = mm_res.prompt_updates
|
||||
|
||||
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
|
||||
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
|
||||
|
||||
@@ -789,101 +851,33 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
),
|
||||
]
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
"""Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`."""
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
|
||||
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
|
||||
if use_audio_in_video and "video" in mm_counts:
|
||||
if mm_counts.get("audio", 0) < mm_counts["video"]:
|
||||
raise ValueError(
|
||||
"Video doesn't have audio track with `audio_in_video=True`"
|
||||
)
|
||||
mm_counts["audio"] -= mm_counts["video"]
|
||||
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
mm_data = dict(processor_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
|
||||
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
|
||||
if audios:
|
||||
# NOTE: Qwen2.5-Omni processor accept "audio"
|
||||
mm_data["audio"] = audios
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
)
|
||||
elif (
|
||||
hf_processor_mm_kwargs.get("use_audio_in_video")
|
||||
and "audio" not in mm_data
|
||||
and mm_data.get("videos")
|
||||
):
|
||||
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
|
||||
# and populated mm_data["audio"] itself before delegating here,
|
||||
# in which case this isn't a genuine "no audio" case. Likewise,
|
||||
# mm_data can be empty on a multimodal-processor-cache hit, where
|
||||
# there's nothing to (re-)process this call and the real result
|
||||
# comes from the cache — not a genuine "no audio" case either.
|
||||
raise ValueError(
|
||||
"Video doesn't have audio track with `audio_in_video=True`"
|
||||
)
|
||||
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
if mm_data.get("videos") and (
|
||||
merged.keys() & {"size", "min_pixels", "max_pixels"}
|
||||
):
|
||||
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||
video_size = dict(self.info.get_hf_processor().video_processor.size)
|
||||
size_override = merged.get("size")
|
||||
if size_override is not None:
|
||||
video_size = video_size | size_override
|
||||
min_pixels = merged.get("min_pixels")
|
||||
if min_pixels is not None:
|
||||
video_size["shortest_edge"] = min_pixels
|
||||
max_pixels = merged.get("max_pixels")
|
||||
if max_pixels is not None:
|
||||
video_size["longest_edge"] = max_pixels
|
||||
hf_processor_mm_kwargs["size"] = video_size
|
||||
|
||||
hf_inputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
input_features = hf_inputs.pop("input_features", None)
|
||||
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
|
||||
if "input_audio_features" not in hf_inputs and input_features is not None:
|
||||
input_features = processed_data.pop("input_features", None)
|
||||
feature_attention_mask = processed_data.get("feature_attention_mask", None)
|
||||
if "input_audio_features" not in processed_data and input_features is not None:
|
||||
if feature_attention_mask is not None:
|
||||
input_features = input_features.permute(0, 2, 1)[
|
||||
feature_attention_mask.bool()
|
||||
].permute(1, 0)
|
||||
hf_inputs["input_audio_features"] = input_features
|
||||
processed_data["input_audio_features"] = input_features
|
||||
if (
|
||||
"audio_feature_lengths" not in hf_inputs
|
||||
"audio_feature_lengths" not in processed_data
|
||||
and feature_attention_mask is not None
|
||||
):
|
||||
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
|
||||
processed_data["audio_feature_lengths"] = feature_attention_mask.sum(-1)
|
||||
|
||||
video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
|
||||
video_second_per_grid = processed_data.get("video_second_per_grid", None)
|
||||
if video_second_per_grid is not None:
|
||||
hf_inputs["second_per_grid_ts"] = video_second_per_grid
|
||||
processed_data["second_per_grid_ts"] = video_second_per_grid
|
||||
|
||||
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
|
||||
hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
|
||||
|
||||
processed_data = hf_inputs
|
||||
processed_data.update(passthrough_data)
|
||||
processed_data.pop("input_ids")
|
||||
use_audio_in_video = hf_kwargs.get("use_audio_in_video", False)
|
||||
processed_data["use_audio_in_video"] = torch.tensor(use_audio_in_video)
|
||||
processed_data.pop("input_ids", None)
|
||||
|
||||
return processed_data
|
||||
|
||||
|
||||
@@ -59,6 +59,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -233,29 +234,25 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
|
||||
|
||||
|
||||
class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
if audios:
|
||||
mm_data["audio"] = audios
|
||||
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
truncation=True,
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
|
||||
return hf_inputs._replace(
|
||||
hf_kwargs=dict(
|
||||
hf_inputs.hf_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
truncation=True,
|
||||
)
|
||||
)
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -37,11 +37,10 @@ from vllm.model_executor.models.qwen3_asr import (
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.inputs import MultiModalKwargsOptionalItems
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseDummyInputsBuilder,
|
||||
MultiModalPromptUpdates,
|
||||
MultiModalProcessingResult,
|
||||
PlaceholderFeaturesInfo,
|
||||
cached_encode,
|
||||
)
|
||||
@@ -114,10 +113,10 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
prompt_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsOptionalItems,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
mm_res: MultiModalProcessingResult,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
mm_kwargs = mm_res.kwargs
|
||||
|
||||
audios = mm_kwargs.get("audio", [])
|
||||
assert len(audios) == 1, (
|
||||
f"Expected only one audio input for realtime, got {len(audios)}"
|
||||
@@ -147,7 +146,7 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
|
||||
# Find the audio_pad token position and expand it to audio_len tokens
|
||||
expanded_ids = list[int]()
|
||||
pad_start_idx = -1
|
||||
for i, tid in enumerate(prompt_ids):
|
||||
for i, tid in enumerate(mm_res.prompt_ids):
|
||||
if tid == audio_pad_id and pad_start_idx == -1:
|
||||
pad_start_idx = i
|
||||
expanded_ids.extend([audio_pad_id] * audio_len)
|
||||
|
||||
@@ -73,6 +73,8 @@ from vllm.multimodal.inputs import (
|
||||
)
|
||||
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems
|
||||
from vllm.multimodal.processing.processor import (
|
||||
HFMultiModalInputs,
|
||||
MultiModalProcessingResult,
|
||||
MultiModalPromptUpdates,
|
||||
PlaceholderFeaturesInfo,
|
||||
PromptReplacement,
|
||||
@@ -1219,13 +1221,26 @@ Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder
|
||||
class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
Qwen2_5OmniThinkerMultiModalProcessor,
|
||||
):
|
||||
def _get_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
audios = processor_data.get("audios")
|
||||
if mm_items.get_count("audio", strict=False):
|
||||
# TODO(Isotr0py): Remove this patch after upstream fix PR
|
||||
# released and Transformers version update:
|
||||
# https://github.com/huggingface/transformers/pull/41473
|
||||
hf_inputs = hf_inputs._replace(
|
||||
hf_kwargs={
|
||||
**hf_inputs.hf_kwargs,
|
||||
"audio_kwargs": dict(hf_inputs.hf_kwargs.get("audio_kwargs") or {}),
|
||||
"text_kwargs": dict(hf_inputs.hf_kwargs.get("text_kwargs") or {}),
|
||||
}
|
||||
)
|
||||
|
||||
audios = hf_inputs.hf_data.get("audio")
|
||||
if audios:
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
hop_length = feature_extractor.hop_length
|
||||
@@ -1239,47 +1254,30 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
|
||||
# To make sure the cache works with padding=True, we pre-padded
|
||||
# the audio to a multiple of hop_length.
|
||||
processor_data = dict(processor_data)
|
||||
processor_data["audios"] = [
|
||||
hf_inputs.hf_data["audio"] = [
|
||||
pad_to_hop_length(audio)
|
||||
if isinstance(audio, np.ndarray)
|
||||
else (pad_to_hop_length(audio[0]), audio[1])
|
||||
for audio in audios
|
||||
]
|
||||
|
||||
return processor_data, passthrough_data
|
||||
return hf_inputs
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if mm_items.get_all_counts().get("audio", 0):
|
||||
# TODO(Isotr0py): Remove this patch after upstream fix PR
|
||||
# released and Transformers version update:
|
||||
# https://github.com/huggingface/transformers/pull/41473
|
||||
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||
hf_processor_mm_kwargs["audio_kwargs"] = dict(
|
||||
hf_processor_mm_kwargs.get("audio_kwargs") or {}
|
||||
)
|
||||
hf_processor_mm_kwargs["text_kwargs"] = dict(
|
||||
hf_processor_mm_kwargs.get("text_kwargs") or {}
|
||||
)
|
||||
|
||||
processed_data = super()._apply_hf_processor_main(
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
processed_data = super()._postprocess_hf_mm_data(
|
||||
mm_data, hf_processor_mm_kwargs, processed_data
|
||||
)
|
||||
|
||||
if (
|
||||
"audio_feature_lengths" in processed_data
|
||||
and "feature_attention_mask" in processed_data
|
||||
):
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
processor_data, _ = self._get_hf_mm_data(valid_mm_items)
|
||||
audios = processor_data.get("audios", [])
|
||||
audios = mm_data.get("audio", [])
|
||||
if audios:
|
||||
feature_extractor = self.info.get_feature_extractor(
|
||||
**hf_processor_mm_kwargs
|
||||
@@ -1312,12 +1310,14 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
prompt_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsItems,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
mm_res: MultiModalProcessingResult,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
"""Qwen3-Omni reimplements this function to handle `use_audio_in_video`."""
|
||||
mm_item_counts = mm_items.get_all_counts()
|
||||
prompt_ids = mm_res.prompt_ids
|
||||
mm_kwargs = mm_res.kwargs
|
||||
mm_prompt_updates = mm_res.prompt_updates
|
||||
|
||||
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
|
||||
|
||||
use_audio_in_video = False
|
||||
|
||||
@@ -1327,27 +1327,25 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
|
||||
return proc_impl is not None and proc_impl is not mixin_impl
|
||||
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
if not hf_data:
|
||||
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
# Separate video processing from image processing. Because the videos
|
||||
# are processed into several image patches
|
||||
video_input_ids_lst: list[list[int]] = []
|
||||
if videos := mm_data.pop("videos", []):
|
||||
if videos := hf_data.pop("videos", []):
|
||||
video_grid_thw_lst = []
|
||||
pixel_values_videos_lst = []
|
||||
timestamps_per_video = []
|
||||
@@ -1370,10 +1368,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
|
||||
# NOTE: a copy of is created to update do_sample_frames,
|
||||
# otherwise mm_hash for the object will be incorrect.
|
||||
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(
|
||||
hf_processor_mm_kwargs, modality="video"
|
||||
)
|
||||
video_mm_kwargs = dict(**hf_kwargs)
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")
|
||||
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
|
||||
video_size = dict(self.info.get_video_processor().size)
|
||||
size_override = merged.get("size")
|
||||
@@ -1485,13 +1481,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
# fps/num_frames are video-only kwargs already consumed by the loop;
|
||||
# exclude them so the text/image processor call below never gets a list.
|
||||
non_video_mm_kwargs = {
|
||||
k: v
|
||||
for k, v in hf_processor_mm_kwargs.items()
|
||||
if k not in ("fps", "num_frames")
|
||||
k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames")
|
||||
}
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**non_video_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_data,
|
||||
non_video_mm_kwargs,
|
||||
)
|
||||
|
||||
@@ -1516,9 +1510,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
processed_data["input_ids"] = [expanded_ids]
|
||||
|
||||
processed_data.update(video_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -202,7 +202,9 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing
|
||||
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
|
||||
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
_, passthrough_data = self._get_hf_mm_data(mm_items)
|
||||
passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_processor_mm_kwargs
|
||||
).passthrough_data
|
||||
mm_processed_data = BatchFeature(
|
||||
{
|
||||
k: torch.as_tensor(v).unsqueeze(0)
|
||||
|
||||
@@ -219,18 +219,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
Subclasses add the strategy for locating placeholders in the prompt.
|
||||
"""
|
||||
|
||||
def _get_hf_mm_data(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""Rename the parser's `audios` key to the `audio` argument HF audio
|
||||
processors take.
|
||||
"""
|
||||
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
||||
if self.info._is_audio_model() and "audios" in processor_data:
|
||||
processor_data["audio"] = processor_data.pop("audios")
|
||||
return processor_data, passthrough_data
|
||||
|
||||
def _get_modality_field_names(self, modality: str) -> set[str]:
|
||||
"""Names of the fields the sub-processor for `modality` produces."""
|
||||
# TODO: use else branch only once huggingface/transformers#44394 lands.
|
||||
@@ -353,6 +341,9 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
|
||||
return mm_fields
|
||||
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _unpad_audios(
|
||||
self,
|
||||
hf_inputs: "BatchFeature",
|
||||
@@ -597,21 +588,31 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
# work for Transformers. The vision path has logic tied to
|
||||
# `mm_tokens_per_modality` in _apply_vision()
|
||||
hf_processor_mm_kwargs = {
|
||||
**hf_processor_mm_kwargs,
|
||||
# vLLM needs the untruncated sequence to keep placeholder
|
||||
# tokens aligned. Note that the text inputs are just dummy
|
||||
# text, not the original prompt. The original prompt is
|
||||
# already tokenized by the renderer.
|
||||
"truncation": False,
|
||||
# HF processors only accept text, and the decoded string already
|
||||
# contains any special tokens, so don't let them be added again
|
||||
# (the HF processor call disables `add_special_tokens`).
|
||||
"add_special_tokens": False,
|
||||
**hf_processor_mm_kwargs,
|
||||
}
|
||||
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
processor_data, _, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_processor_mm_kwargs
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
# The real prompt is fed to the processor below and the placeholder
|
||||
# ranges are read back out of its output, so the dummy text meant for
|
||||
# media-only callers is discarded here.
|
||||
processor_data.pop("text", None)
|
||||
has_mm_data = any(processor_data.values())
|
||||
|
||||
# Ask which modality owns each token of the expanded prompt, which
|
||||
# is the only marking of the tokens an expansion adds around an item
|
||||
if any(processor_data.values()):
|
||||
if has_mm_data:
|
||||
processor_data = {**processor_data, "return_mm_token_type_ids": True}
|
||||
|
||||
try:
|
||||
@@ -621,7 +622,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
except ValueError:
|
||||
if any(processor_data.values()):
|
||||
if has_mm_data:
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
@@ -792,31 +793,37 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
except (AttributeError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
|
||||
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
if not hf_data:
|
||||
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
prompt_text = hf_data.pop("text")
|
||||
assert isinstance(prompt_text, str)
|
||||
|
||||
# Ask for the replacement each placeholder expands to, and record it as
|
||||
# per-item fields: its token ids, and the tokens or patches behind them
|
||||
if has_mm_data := any(mm_data.values()):
|
||||
mm_data = {**mm_data, "return_text_replacement_offsets": True}
|
||||
if has_mm_data := any(hf_data.values()):
|
||||
hf_data = {**hf_data, "return_text_replacement_offsets": True}
|
||||
|
||||
try:
|
||||
hf_inputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
dict(text=prompt_text, **hf_data),
|
||||
hf_kwargs,
|
||||
)
|
||||
except ValueError:
|
||||
if any(mm_data.values()):
|
||||
if any(hf_data.values()):
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
@@ -825,7 +832,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(hf_inputs)
|
||||
self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)
|
||||
self._unpad_audios(hf_inputs, hf_data, hf_kwargs)
|
||||
|
||||
# Drop the inputs the model would reject
|
||||
hf_inputs.pop("mm_token_type_ids", None)
|
||||
@@ -845,9 +852,10 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"so it can be fixed, and install `transformers<5.15.0` in the "
|
||||
"meantime to locate placeholders in the expanded prompt instead."
|
||||
)
|
||||
hf_inputs.update(passthrough_data)
|
||||
hf_inputs.pop("input_ids", None)
|
||||
return hf_inputs
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, hf_inputs
|
||||
)
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
replacements = defaultdict[str, list[list[int]]](list)
|
||||
@@ -865,7 +873,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
)
|
||||
if modality == "image":
|
||||
hf_inputs["num_image_patches"] = self._get_num_image_patches(
|
||||
hf_inputs, mm_data, len(seqs)
|
||||
hf_inputs, hf_data, len(seqs)
|
||||
)
|
||||
elif modality == "audio":
|
||||
counts = []
|
||||
@@ -874,10 +882,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
|
||||
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
|
||||
|
||||
hf_inputs.update(passthrough_data)
|
||||
hf_inputs.pop("input_ids")
|
||||
|
||||
return hf_inputs
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, hf_inputs
|
||||
)
|
||||
|
||||
|
||||
# From this version on, a processor reporting no offsets is an error rather than a
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
|
||||
@@ -190,72 +191,32 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
|
||||
|
||||
|
||||
class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
audios = mm_data.get("audios", [])
|
||||
assert isinstance(audios, list)
|
||||
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
include_audio_num_chunks=True,
|
||||
)
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
if processor_data:
|
||||
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
# The Ultravox processor accepts "audios" instead of "audio"
|
||||
hf_data = hf_inputs.hf_data
|
||||
if "audio" in hf_data:
|
||||
hf_data["audios"] = hf_data.pop("audio")
|
||||
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
|
||||
normalized_kwargs = dict(hf_inputs.hf_kwargs)
|
||||
# The remote processor already hardcodes truncation=False and forwards
|
||||
# its kwargs to the audio processor, where another value would collide.
|
||||
normalized_kwargs.pop("truncation", None)
|
||||
return hf_inputs._replace(
|
||||
hf_kwargs=dict(
|
||||
normalized_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
include_audio_num_chunks=True,
|
||||
)
|
||||
|
||||
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
|
||||
if prompt_text is not None:
|
||||
processor_data = dict(text=prompt_text, **processor_data)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
def patched_call(**kwargs) -> BatchFeature:
|
||||
# The remote UltravoxProcessor does not support the
|
||||
# `truncation` kwarg injected by `call_hf_processor`: it
|
||||
# passes `truncation` to its audio processor by itself while
|
||||
# also forwarding `**kwargs` there, causing a duplicate
|
||||
# keyword argument error. Its hardcoded `truncation=False`
|
||||
# already matches vLLM's intended behavior.
|
||||
kwargs.pop("truncation", None)
|
||||
|
||||
return hf_processor(**kwargs)
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
patched_call,
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
else:
|
||||
processed_data = BatchFeature(dict(passthrough_data))
|
||||
|
||||
return self._postprocess_hf_mm_data(
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
processed_data,
|
||||
)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
|
||||
@@ -75,7 +75,7 @@ from vllm.multimodal.parse import (
|
||||
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
|
||||
from vllm.multimodal.processing.context import TimingContext
|
||||
from vllm.multimodal.processing.inputs import ProcessorInputs
|
||||
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
|
||||
from vllm.multimodal.processing.processor import MultiModalProcessingResult
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import (
|
||||
BASE_SIZE,
|
||||
CROP_MODE,
|
||||
@@ -225,7 +225,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
) -> MultiModalProcessingResult:
|
||||
# The processor logic differs for single-image (crop) vs multi-image
|
||||
# (no crop) requests. The processing cache assumes per-item output is
|
||||
# invariant of how many images are passed per prompt, so we only cache
|
||||
|
||||
@@ -16,7 +16,7 @@ from mistral_common.protocol.instruct.messages import UserMessage
|
||||
from mistral_common.protocol.instruct.request import ChatCompletionRequest
|
||||
from mistral_common.protocol.transcription.request import TranscriptionRequest
|
||||
from mistral_common.tokens.tokenizers.audio import Audio
|
||||
from transformers import BatchFeature, WhisperConfig
|
||||
from transformers import WhisperConfig
|
||||
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
@@ -36,7 +36,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
MultiModalKwargsOptionalItems,
|
||||
NestedTensors,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
@@ -48,7 +47,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
MultiModalPromptUpdates,
|
||||
HFMultiModalInputs,
|
||||
MultiModalProcessingResult,
|
||||
PlaceholderFeaturesInfo,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
@@ -61,7 +61,6 @@ from vllm.transformers_utils.processors.voxtral import (
|
||||
MistralCommonFeatureExtractor,
|
||||
MistralCommonVoxtralProcessor,
|
||||
)
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription
|
||||
from .utils import init_vllm_registered_model, maybe_prefix
|
||||
@@ -209,21 +208,19 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
prompt_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsOptionalItems,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
mm_res: MultiModalProcessingResult,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
mm_item_counts = mm_items.get_all_counts()
|
||||
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
|
||||
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
|
||||
self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
|
||||
self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
|
||||
|
||||
mm_placeholders = self._find_mm_placeholders(
|
||||
prompt_ids,
|
||||
mm_prompt_updates,
|
||||
mm_res.prompt_ids,
|
||||
mm_res.prompt_updates,
|
||||
)
|
||||
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
|
||||
|
||||
return prompt_ids, mm_placeholders
|
||||
return mm_res.prompt_ids, mm_placeholders
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
@@ -241,38 +238,20 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
||||
# skip validation here
|
||||
pass
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
|
||||
if audios:
|
||||
# MistralCommonVoxtralProcessor accepts "audio"
|
||||
mm_data["audio"] = audios
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
# Avoid padding issue
|
||||
hf_processor_mm_kwargs = dict(**hf_processor_mm_kwargs, return_tensors=None)
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
# Missing batch dimension
|
||||
input_ids = processed_data.get("input_ids")
|
||||
if is_list_of(input_ids, int):
|
||||
processed_data["input_ids"] = [input_ids]
|
||||
|
||||
return processed_data
|
||||
return hf_inputs._replace(
|
||||
hf_kwargs=dict(hf_inputs.hf_kwargs, return_tensors=None)
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -28,11 +28,10 @@ from vllm.model_executor.models.voxtral import (
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.inputs import MultiModalKwargsOptionalItems
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import BaseDummyInputsBuilder
|
||||
from vllm.multimodal.processing.processor import (
|
||||
MultiModalPromptUpdates,
|
||||
MultiModalProcessingResult,
|
||||
PlaceholderFeaturesInfo,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -60,10 +59,10 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
prompt_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsOptionalItems,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
mm_res: MultiModalProcessingResult,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
mm_kwargs = mm_res.kwargs
|
||||
|
||||
# there are no placeholder audio tokens for streaming
|
||||
# so we need to build the place placeholder positions manually
|
||||
|
||||
@@ -86,7 +85,7 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
|
||||
* [0], # only used for length computation, so we can take dummy inputs
|
||||
is_embed=None,
|
||||
)
|
||||
return prompt_ids, {"audio": [features_info]}
|
||||
return mm_res.prompt_ids, {"audio": [features_info]}
|
||||
|
||||
|
||||
class TimeEmbedding(torch.nn.Module):
|
||||
|
||||
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.utils.jsontree import json_map_leaves
|
||||
@@ -743,6 +744,9 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
|
||||
|
||||
|
||||
class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo]):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def create_encoder_prompt(
|
||||
self,
|
||||
prompt: list[int],
|
||||
@@ -754,37 +758,21 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
|
||||
# for encoder profiling.
|
||||
return [0]
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
|
||||
return hf_inputs._replace(
|
||||
hf_kwargs=dict(
|
||||
hf_inputs.hf_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
)
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "labels" in processed_data:
|
||||
processed_data["input_ids"] = processed_data.pop("labels")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
||||
|
||||
from .video import preprocess_dots3_note_video
|
||||
@@ -630,26 +631,33 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo
|
||||
|
||||
|
||||
class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingInfo]):
|
||||
def _get_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
||||
if "video" not in mm_items:
|
||||
return processor_data, passthrough_data
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
|
||||
videos = mm_items.get_items("video", VideoProcessorItems)
|
||||
raw_videos: list[object] = []
|
||||
for index, item in enumerate(videos.data):
|
||||
if isinstance(item, MediaWithBytes):
|
||||
raw_videos.append(item.original_bytes)
|
||||
else:
|
||||
raw_videos.append(videos.get(index))
|
||||
processor_data = dict(processor_data)
|
||||
processor_data["videos"] = raw_videos
|
||||
return processor_data, passthrough_data
|
||||
# The Dots3Note processor accepts "audios" instead of "audio"
|
||||
hf_data = hf_inputs.hf_data
|
||||
if "audio" in hf_data:
|
||||
hf_data["audios"] = hf_data.pop("audio")
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
if "video" in mm_items:
|
||||
videos = mm_items.get_items("video", VideoProcessorItems)
|
||||
|
||||
raw_videos: list[object] = []
|
||||
for index, item in enumerate(videos.data):
|
||||
if isinstance(item, MediaWithBytes):
|
||||
raw_videos.append(item.original_bytes)
|
||||
else:
|
||||
raw_videos.append(videos.get(index))
|
||||
|
||||
hf_data["videos"] = raw_videos
|
||||
|
||||
return hf_inputs
|
||||
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -182,20 +182,19 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
processor = self.info.get_hf_processor(**hf_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
images = mm_data.get("images") or []
|
||||
audios = mm_data.get("audios") or []
|
||||
audios = mm_data.get("audio") or []
|
||||
if not isinstance(images, list):
|
||||
images = list(cast(Iterable[Any], images))
|
||||
if not isinstance(audios, list):
|
||||
@@ -235,8 +234,9 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
|
||||
data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)
|
||||
|
||||
processed_data = BatchFeature(data=data, tensor_type=None)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
return self._finalize_hf_mm_data(
|
||||
mm_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _tokenize_with_placeholders(
|
||||
self,
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.multimodal.video import (
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
VideoBackend,
|
||||
@@ -307,26 +308,21 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing
|
||||
class MiniMaxM3VLMultiModalProcessor(
|
||||
BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo]
|
||||
):
|
||||
def _apply_hf_processor_main(
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
hf_data = hf_inputs.hf_data
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
# With ``video_needs_metadata=True`` each video arrives as a
|
||||
# ``(frames, metadata)`` tuple. Split the frames back out and forward the
|
||||
# metadata as ``VideoMetadata`` so the processor emits timestamps.
|
||||
videos = cast(list | None, mm_data.get("videos"))
|
||||
videos = cast(list | None, hf_data.get("videos"))
|
||||
video_metadata: list[VideoMetadata] | None = None
|
||||
if videos:
|
||||
frames_only = []
|
||||
@@ -346,21 +342,25 @@ class MiniMaxM3VLMultiModalProcessor(
|
||||
# stays consistent with _get_prompt_updates.
|
||||
meta.setdefault("total_num_frames", len(frames))
|
||||
video_metadata.append(VideoMetadata(**meta))
|
||||
mm_data["videos"] = frames_only
|
||||
hf_data["videos"] = frames_only
|
||||
|
||||
# Override the video processor's default do_resize=False (set for a
|
||||
# pre-resized pipeline) to True for vLLM's raw-frame inputs.
|
||||
merged = dict(do_resize=True, **hf_processor_mm_kwargs)
|
||||
data = dict(text=prompt_text, **mm_data)
|
||||
if video_metadata is not None:
|
||||
data["video_metadata"] = video_metadata
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
data,
|
||||
hf_data["video_metadata"] = video_metadata
|
||||
|
||||
return hf_inputs
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override the video processor's default for vLLM's raw-frame inputs.
|
||||
merged = dict(do_resize=True, **hf_kwargs)
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
hf_data,
|
||||
merged,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -321,12 +321,6 @@ class InputProcessingContext:
|
||||
|
||||
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
|
||||
|
||||
# vLLM needs the full untruncated sequence to keep multi-modal
|
||||
# placeholder tokens aligned; note that the text inputs in
|
||||
# call_hf_processor are just dummy text, not the original prompt.
|
||||
# The original prompt is already tokenized by the renderer.
|
||||
merged_kwargs.setdefault("truncation", False)
|
||||
|
||||
allowed_kwargs = get_allowed_kwarg_only_overrides(
|
||||
hf_processor,
|
||||
merged_kwargs,
|
||||
@@ -426,7 +420,7 @@ class BaseProcessingInfo:
|
||||
def get_data_parser(self) -> MultiModalDataParser:
|
||||
"""Constructs a parser to preprocess multi-modal data items
|
||||
before passing them to
|
||||
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
|
||||
[`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].
|
||||
|
||||
You can support additional modalities by creating a subclass
|
||||
of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
|
||||
@@ -506,7 +500,7 @@ class BaseProcessingInfo:
|
||||
"""Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
|
||||
to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
|
||||
before passing them to
|
||||
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
|
||||
[`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].
|
||||
"""
|
||||
mm_items = self.data_parser.parse_mm_data(mm_data)
|
||||
|
||||
|
||||
@@ -1008,7 +1008,20 @@ For an item `MultiModalPromptUpdates[k][i]`,
|
||||
_I = TypeVar("_I", bound=BaseProcessingInfo)
|
||||
|
||||
|
||||
class MultiModalProcessingInfo(NamedTuple):
|
||||
class HFMultiModalInputs(NamedTuple):
|
||||
hf_data: dict[str, object]
|
||||
"""The data inputs to the HF processor."""
|
||||
|
||||
hf_kwargs: Mapping[str, object]
|
||||
"""The non-data inputs to the HF processor."""
|
||||
|
||||
passthrough_data: dict[str, object]
|
||||
"""The inputs to assign to the output of the HF processor,
|
||||
instead of passing them to the HF processor."""
|
||||
|
||||
|
||||
class MultiModalProcessingResult(NamedTuple):
|
||||
prompt_ids: list[int]
|
||||
kwargs: MultiModalKwargsOptionalItems
|
||||
hashes: MultiModalHashes
|
||||
prompt_updates: MultiModalPromptUpdates
|
||||
@@ -1137,23 +1150,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
|
||||
return find_mm_placeholders(new_token_ids, mm_prompt_updates)
|
||||
|
||||
def _get_hf_mm_data(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""Extract processor and passthrough data from multi-modal items."""
|
||||
processor_data = dict[str, object]()
|
||||
passthrough_data = dict[str, object]()
|
||||
|
||||
for items in mm_items.values():
|
||||
processor_data.update(items.get_processor_data())
|
||||
passthrough_data.update(items.get_passthrough_data())
|
||||
|
||||
return processor_data, passthrough_data
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None:
|
||||
"""Get the text to pass to the HF processor alongside the multi-modal
|
||||
data.
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
|
||||
"""Get the text to pass to the HF processor alongside the multi-modal data.
|
||||
|
||||
By default, no text is passed. If the HF processor requires that
|
||||
text and multi-modal items correspond to each other, you should
|
||||
@@ -1162,29 +1160,53 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"""
|
||||
return None
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""Pre-process the multi-modal data and HF processor keyword arguments
|
||||
before they are passed to the HF processor.
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
"""Extract the multi-modal data and corresponding keyword arguments
|
||||
for HF processing.
|
||||
|
||||
By default, both are returned as-is. If the HF processor expects the
|
||||
multi-modal data under different keys than those provided by the
|
||||
multi-modal items (e.g. `audio` instead of `audios`), or requires
|
||||
additional keyword arguments (e.g. `sampling_rate`), you should
|
||||
override this method.
|
||||
If the HF processor expects the multi-modal data under different keys
|
||||
than those provided by the multi-modal items, or requires additional
|
||||
keyword arguments, you should override this method.
|
||||
"""
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
hf_data = dict[str, object]()
|
||||
passthrough_data = dict[str, object]()
|
||||
|
||||
for items in mm_items.values():
|
||||
if not items:
|
||||
continue
|
||||
|
||||
hf_data.update(items.get_processor_data())
|
||||
passthrough_data.update(items.get_passthrough_data())
|
||||
|
||||
if hf_data:
|
||||
prompt_text = self._get_hf_mm_text(mm_items.get_all_counts())
|
||||
if prompt_text is not None:
|
||||
hf_data = dict(text=prompt_text, **hf_data)
|
||||
|
||||
# vLLM needs the untruncated sequence to keep placeholder
|
||||
# tokens aligned. Note that the text inputs are just dummy
|
||||
# text, not the original prompt. The original prompt is
|
||||
# already tokenized by the renderer.
|
||||
hf_kwargs = {"truncation": False, **hf_kwargs}
|
||||
|
||||
# HF processors accept (images, text, videos, audio)
|
||||
# so we need to remap vLLM's "audios" -> HF "audio"
|
||||
if "audios" in hf_data:
|
||||
hf_data["audio"] = hf_data.pop("audios")
|
||||
|
||||
return HFMultiModalInputs(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_inputs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
"""Post-process the output of the HF processor.
|
||||
"""Post-process the combined processor and passthrough data.
|
||||
|
||||
By default, the output is returned as-is. If you need to modify the
|
||||
output of the HF processor before it is converted into multi-modal
|
||||
@@ -1192,41 +1214,51 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"""
|
||||
return processed_data
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Call the HF processor on the normalized multi-modal inputs."""
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
hf_data,
|
||||
hf_kwargs,
|
||||
)
|
||||
|
||||
def _finalize_hf_mm_data(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
passthrough_data: Mapping[str, object],
|
||||
processed_data: BatchFeature | None = None,
|
||||
) -> BatchFeature:
|
||||
"""Merge passthrough fields and apply model-specific postprocessing."""
|
||||
if processed_data is None:
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
processed_data = BatchFeature()
|
||||
|
||||
processed_data.update(passthrough_data)
|
||||
return self._postprocess_hf_mm_data(hf_data, hf_kwargs, processed_data)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Apply the HF processor on the multi-modal data."""
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if processor_data:
|
||||
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
|
||||
processor_data, hf_processor_mm_kwargs
|
||||
)
|
||||
|
||||
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
|
||||
if prompt_text is not None:
|
||||
processor_data = dict(text=prompt_text, **processor_data)
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
if hf_data:
|
||||
processed_data = self._call_hf_processor(hf_data, hf_kwargs)
|
||||
else:
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
processed_data = None
|
||||
|
||||
processed_data = BatchFeature(dict(passthrough_data))
|
||||
|
||||
return self._postprocess_hf_mm_data(
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
processed_data,
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||
@@ -1261,6 +1293,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
mm_missing_data = {}
|
||||
for modality, idxs in mm_missing_idxs.items():
|
||||
if not idxs:
|
||||
continue
|
||||
|
||||
missing_modality_data = []
|
||||
for idx in idxs:
|
||||
data = mm_data_items[modality][idx]
|
||||
@@ -1271,6 +1306,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
)
|
||||
else:
|
||||
missing_modality_data.append(data)
|
||||
|
||||
mm_missing_data[modality] = missing_modality_data
|
||||
|
||||
mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False)
|
||||
@@ -1342,11 +1378,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
) -> MultiModalProcessingResult:
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
mm_processed_data = self._apply_hf_processor_main(
|
||||
mm_items=inputs.mm_data_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
hf_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
@@ -1369,26 +1405,28 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
mm_info = MultiModalProcessingInfo(
|
||||
return MultiModalProcessingResult(
|
||||
prompt_ids=self._postprocess_prompt(inputs.prompt),
|
||||
kwargs=mm_kwargs,
|
||||
hashes=mm_hashes,
|
||||
prompt_updates=mm_prompt_updates,
|
||||
)
|
||||
|
||||
return mm_info
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
) -> MultiModalProcessingResult:
|
||||
"""Apply the HF processor on the full prompt text,
|
||||
caching the results and reusing cached results.
|
||||
"""
|
||||
cache = self.cache
|
||||
has_passthrough_data = any(
|
||||
len(items.get_passthrough_data()) > 0
|
||||
for items in inputs.mm_data_items.values()
|
||||
)
|
||||
|
||||
_, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items)
|
||||
if cache is None or passthrough_data:
|
||||
if cache is None or has_passthrough_data:
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
@@ -1410,7 +1448,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
mm_missing_processed_data = self._apply_hf_processor_main(
|
||||
mm_items=mm_missing_data_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
hf_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
@@ -1435,14 +1473,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_missing_prompt_updates=mm_missing_prompt_updates,
|
||||
)
|
||||
|
||||
mm_info = MultiModalProcessingInfo(
|
||||
return MultiModalProcessingResult(
|
||||
prompt_ids=self._postprocess_prompt(inputs.prompt),
|
||||
kwargs=mm_kwargs,
|
||||
hashes=mm_hashes,
|
||||
prompt_updates=mm_prompt_updates,
|
||||
)
|
||||
|
||||
return mm_info
|
||||
|
||||
def _apply_token_matches(
|
||||
self,
|
||||
prompt: list[int],
|
||||
@@ -1670,17 +1707,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
prompt_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsOptionalItems,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
mm_res: MultiModalProcessingResult,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
mm_item_counts = mm_items.get_all_counts()
|
||||
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
|
||||
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
|
||||
self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
|
||||
self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
|
||||
|
||||
prompt_ids, mm_placeholders = self._apply_prompt_updates(
|
||||
prompt_ids,
|
||||
mm_prompt_updates,
|
||||
mm_res.prompt_ids,
|
||||
mm_res.prompt_updates,
|
||||
)
|
||||
|
||||
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
|
||||
@@ -1705,15 +1740,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
4. Extract information about the placeholder tokens from the
|
||||
processed token IDs.
|
||||
"""
|
||||
prompt_ids = self._postprocess_prompt(inputs.prompt)
|
||||
mm_info = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
mm_res = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
with timing_ctx.record("apply_prompt_updates"):
|
||||
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
|
||||
mm_items=inputs.mm_data_items,
|
||||
prompt_ids=prompt_ids,
|
||||
mm_kwargs=mm_info.kwargs,
|
||||
mm_prompt_updates=mm_info.prompt_updates,
|
||||
mm_res=mm_res,
|
||||
)
|
||||
|
||||
mm_placeholder_ranges = {
|
||||
@@ -1723,8 +1755,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
return mm_input(
|
||||
prompt_token_ids=prompt_ids,
|
||||
mm_kwargs=mm_info.kwargs,
|
||||
mm_hashes=mm_info.hashes,
|
||||
mm_kwargs=mm_res.kwargs,
|
||||
mm_hashes=mm_res.hashes,
|
||||
mm_placeholders=mm_placeholder_ranges,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user