[MM] Further cleanup _apply_hf_processor_main (#53610)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-16 10:21:00 -07:00
committed by GitHub
parent 6ca2b23e22
commit 0b9e018fb9
74 changed files with 1048 additions and 1323 deletions
+1 -13
View File
@@ -374,21 +374,9 @@ class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):
class AudioFlamingo3MultiModalProcessor(
BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", None)
if audios is not None:
mm_data["audio"] = audios
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
@@ -119,7 +119,7 @@ class BailingMoeV3VLDummyInputsBuilder(
class BailingMoeV3VLMultiModalProcessor(
BaseMultiModalProcessor[BailingMoeV3VLProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_prompt_updates(
+1 -1
View File
@@ -222,7 +222,7 @@ class Cohere2VisionDummyInputsBuilder(
class Cohere2VisionMultiModalProcessor(
BaseMultiModalProcessor[Cohere2VisionProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+18 -28
View File
@@ -8,7 +8,7 @@ from typing import Any, ClassVar
import torch
import torch.nn.functional as F
from torch import nn
from transformers import BatchFeature, PretrainedConfig
from transformers import PretrainedConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
@@ -50,7 +50,10 @@ from vllm.multimodal.processing import (
PromptUpdate,
TimingContext,
)
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
from vllm.multimodal.processing.processor import (
HFMultiModalInputs,
MultiModalProcessingResult,
)
from vllm.renderers import TokenizeParams
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.transformers_utils.processors.cohere_asr import (
@@ -1962,42 +1965,29 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
) -> list[int]:
return [0]
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
return hf_inputs._replace(
hf_kwargs=dict(
hf_inputs.hf_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
# Dithering injects noise into the extracted features, so the
# feature extractor is not a pure function of its input. Since the
# processing cache assumes that processor outputs are invariant
+7 -7
View File
@@ -157,15 +157,14 @@ class ColModernVBertMultiModalProcessor(
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
@@ -193,8 +192,9 @@ class ColModernVBertMultiModalProcessor(
result.update(image_outputs)
processed_data = result
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
mm_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+7 -18
View File
@@ -20,7 +20,7 @@ from collections.abc import Iterable, Mapping
import torch
import torch.nn as nn
from transformers import BatchFeature, PaliGemmaProcessor
from transformers import PaliGemmaProcessor
from vllm.config import VllmConfig
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing.processor import HFMultiModalInputs
from .interfaces import SupportsLateInteraction
from .interfaces_base import default_pooling_type
@@ -59,31 +60,19 @@ class ColPaliProcessingInfo(PaliGemmaProcessingInfo):
class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
"""Multimodal processor for ColPali."""
def _apply_hf_processor_main(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# The ColPali tokenizer_config.json ships with a small default
# max_length (50) that truncates the 1024 image tokens inserted
# by PaliGemmaProcessor, causing a token-count mismatch.
# vLLM enforces its own max_model_len, so we disable HF
# truncation to keep all image + text tokens intact.
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
dict(**hf_processor_mm_kwargs, truncation=False),
)
processed_data.update(passthrough_data)
return processed_data
return hf_inputs._replace(hf_kwargs=dict(hf_inputs.hf_kwargs, truncation=False))
@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
+11 -26
View File
@@ -46,10 +46,10 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sampling_params import SamplingParams
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.hf import HfTokenizer
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
from vllm.transformers_utils.processors.deepseek_ocr import (
BASE_SIZE,
@@ -287,34 +287,19 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing
class DeepseekOCRMultiModalProcessor(
BaseMultiModalProcessor[DeepseekOCRProcessingInfo]
):
def _apply_hf_processor_main(
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
if "text" in hf_inputs.hf_data:
hf_inputs.hf_data["prompt"] = hf_inputs.hf_data.pop("text")
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
if mm_data:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(prompt=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
else:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
processed_data = tokenizer(
prompt_text, add_special_tokens=True, return_tensors="pt"
)
processed_data.update(passthrough_data)
return processed_data
return hf_inputs
def _get_mm_fields_config(
self,
+11 -27
View File
@@ -45,9 +45,9 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.hf import HfTokenizer
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
from vllm.transformers_utils.processors.deepseek_ocr import (
BASE_SIZE,
@@ -158,35 +158,19 @@ class DeepseekOCR2DummyInputsBuilder(
class DeepseekOCR2MultiModalProcessor(
BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo]
):
def _apply_hf_processor_main(
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
if "text" in hf_inputs.hf_data:
hf_inputs.hf_data["prompt"] = hf_inputs.hf_data.pop("text")
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
if mm_data:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(prompt=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
else:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
processed_data = tokenizer(
prompt_text, add_special_tokens=True, return_tensors="pt"
)
processed_data.update(passthrough_data)
return processed_data
return hf_inputs
def _get_mm_fields_config(
self,
+3 -3
View File
@@ -35,7 +35,7 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
MultiModalProcessingInfo,
MultiModalProcessingResult,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
@@ -233,7 +233,7 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
class DeepseekVL2MultiModalProcessor(
BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
@@ -307,7 +307,7 @@ class DeepseekVL2MultiModalProcessor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
# The processor logic is different for len(images) <= 2 vs > 2
# Since the processing cache assumes that the processor output is
# invariant of how many images are passed per prompt, we only
+50 -57
View File
@@ -71,6 +71,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
cached_encode,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -1112,34 +1113,32 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
pixel_values = pixel_values.to(hf_config.dtype)
return pixel_values
def _apply_hf_processor_main(
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
hf_data = hf_inputs.hf_data
if "images" not in mm_data and "videos" not in mm_data:
return BatchFeature(dict(passthrough_data))
if "images" not in hf_data and "videos" not in hf_data:
return hf_inputs
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data)
if "images" not in mm_data:
mm_data["images"] = []
if "videos" not in mm_data:
mm_data["videos"] = []
prompt_text = hf_data["text"]
hf_data["text"] = [prompt_text]
hf_data.setdefault("images", [])
hf_data.setdefault("videos", [])
# Check if HF processor supports video metadata
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_inputs.hf_kwargs)
supports_video_metadata = getattr(
hf_processor, "supports_video_metadata", False
)
videos = mm_data["videos"]
videos = hf_data["videos"]
assert isinstance(videos, Sequence)
if videos and not supports_video_metadata:
# Old HF processor, unwrap tuple to pure frames
@@ -1147,49 +1146,43 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
"HF processor doesn't support video metadata. "
"Timestamps will NOT be rendered. Please upgrade the model."
)
mm_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos]
hf_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos]
processor_output = self.info.ctx.call_hf_processor(
hf_processor,
dict(
text=[prompt_text], images=mm_data["images"], videos=mm_data["videos"]
),
hf_processor_mm_kwargs,
)
return hf_inputs
def _postprocess_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "images" not in hf_data and "videos" not in hf_data:
return processed_data
# Divide the processor_output into two modalities: image and video.
if processor_output is not None:
pixel_values = processor_output["images"]
if pixel_values is not None:
processor_output["images"] = self._pixel_values_norm(
pixel_values, hf_processor_mm_kwargs
)
for key in list(processor_output.keys()):
if processor_output[key] is None:
del processor_output[key]
continue
if key == "grid_thw":
grid_thw = processor_output["grid_thw"]
pixel_values_all = processor_output["images"]
# Identify elements where the first
# dimension is greater than 1 and
# treat them as the video modality
mask = grid_thw[:, 0] > 1
processor_output["video_grid_thw"] = grid_thw[mask]
processor_output["image_grid_thw"] = grid_thw[~mask]
image_patch_num = (
processor_output["image_grid_thw"].prod(dim=1).sum()
)
processor_output["pixel_values"] = pixel_values_all[
:image_patch_num
]
processor_output["pixel_values_videos"] = pixel_values_all[
image_patch_num:
]
del processor_output["images"]
pixel_values = processed_data["images"]
if pixel_values is not None:
processed_data["images"] = self._pixel_values_norm(pixel_values, hf_kwargs)
for key in list(processed_data.keys()):
if processed_data[key] is None:
del processed_data[key]
continue
if key == "grid_thw":
grid_thw = processed_data["grid_thw"]
pixel_values_all = processed_data["images"]
# Identify elements where the first
# dimension is greater than 1 and
# treat them as the video modality
mask = grid_thw[:, 0] > 1
processed_data["video_grid_thw"] = grid_thw[mask]
processed_data["image_grid_thw"] = grid_thw[~mask]
image_patch_num = processed_data["image_grid_thw"].prod(dim=1).sum()
processed_data["pixel_values"] = pixel_values_all[:image_patch_num]
processed_data["pixel_values_videos"] = pixel_values_all[
image_patch_num:
]
del processed_data["images"]
processed_data = processor_output
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates(
+13 -25
View File
@@ -38,6 +38,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.processors.fireredasr2 import (
FireRedASR2FeatureExtractor,
@@ -228,37 +229,24 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing
class FireRedASR2MultiModalProcessor(
BaseMultiModalProcessor[FireRedASR2ProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
return hf_inputs._replace(
hf_kwargs=dict(
hf_inputs.hf_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+17 -27
View File
@@ -49,7 +49,10 @@ from vllm.multimodal.processing import (
PromptUpdate,
TimingContext,
)
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
from vllm.multimodal.processing.processor import (
HFMultiModalInputs,
MultiModalProcessingResult,
)
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -752,42 +755,29 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
return hf_inputs._replace(
hf_kwargs=dict(
hf_inputs.hf_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
# Dithering injects noise into the extracted features, so the
# feature extractor is not a pure function of its input. Since the
# processing cache assumes that processor outputs are invariant
+12 -11
View File
@@ -616,29 +616,29 @@ class FunAudioChatMultiModalProcessor(
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
tokenizer = self.info.get_tokenizer()
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
audios = mm_data.get("audios", [])
audios = mm_data.get("audio", [])
if not audios:
processed_data = BatchFeature({"input_ids": input_ids})
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
mm_data, hf_kwargs, passthrough_data, processed_data
)
assert isinstance(audios, Sequence)
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
sr = int(feature_extractor.sampling_rate)
min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400)
@@ -690,8 +690,9 @@ class FunAudioChatMultiModalProcessor(
}
processed_data = BatchFeature({"input_ids": input_ids, **mm_inputs})
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
mm_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+1 -1
View File
@@ -263,7 +263,7 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+1 -12
View File
@@ -265,20 +265,9 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
if "audios" in mm_data:
mm_data["audio"] = mm_data.pop("audios")
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
+20 -23
View File
@@ -595,22 +595,25 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_kwargs)
val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
if val is not None and val not in _SUPPORTED_SOFT_TOKENS:
@@ -619,14 +622,12 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
f"Valid values are {_SUPPORTED_SOFT_TOKENS}."
)
mm_data = dict(mm_data)
# ---- VIDEO HANDLING ----
# Gemma4 decomposes video into timestamped image frames.
# Each frame is processed with max_soft_tokens=70 through the
# same vision tower, matching transformers processing_gemma4.py.
video_outputs: dict[str, Any] = {}
if videos := mm_data.pop("videos", []):
if videos := hf_data.pop("videos", []):
assert isinstance(videos, list)
processor = self.info.get_hf_processor()
@@ -656,7 +657,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
timestamps = [idx / fps for idx in frame_indices]
# Process frames as images with max_soft_tokens=70
video_mm_kwargs = dict(hf_processor_mm_kwargs)
video_mm_kwargs = dict(hf_kwargs)
# Override configured image options too: these inputs are video
# frames, whose cache keys exclude images_kwargs.
video_mm_kwargs["images_kwargs"] = {}
@@ -726,18 +727,14 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
"video_timestamps": video_timestamps_per_video,
}
# The processor accepts 'audio' not 'audios'.
if "audios" in mm_data:
mm_data["audio"] = mm_data.pop("audios")
# Warn if any audio waveform exceeds the model's max duration.
if "audio" in mm_data:
if "audio" in hf_data:
processor = self.info.get_hf_processor()
sr = processor.feature_extractor.sampling_rate
max_tokens = processor.audio_seq_length
ms_per_tok = processor.audio_ms_per_token
max_duration_s = max_tokens * ms_per_tok / 1000.0
audios = mm_data["audio"]
audios = hf_data["audio"]
if not isinstance(audios, (list, tuple)):
audios = [audios]
for i, waveform in enumerate(audios):
@@ -760,13 +757,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
# NOTE: This requires a corresponding type annotation on the
# HF side (Gemma4ProcessorKwargs.images_kwargs) so that
# _merge_kwargs routes max_soft_tokens into images_kwargs.
patched_mm_kwargs = dict(hf_processor_mm_kwargs)
patched_mm_kwargs = dict(hf_kwargs)
if val is not None and is_top_level_max_soft_tokens:
patched_mm_kwargs["max_soft_tokens"] = val
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**patched_mm_kwargs),
dict(text=prompt_text, **mm_data),
dict(text=prompt_text, **hf_data),
patched_mm_kwargs,
)
@@ -797,9 +794,9 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
processed_data["input_features_mask"] = unpadded_masks
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+30 -23
View File
@@ -1582,52 +1582,59 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
return prepared_data, prepared_kwargs
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
mm_data = dict(mm_data)
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
processor = self.info.get_hf_processor(**hf_kwargs)
use_direct_path = (
not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
)
if use_direct_path:
prepared_data, prepared_kwargs = self._get_direct_path_inputs(
mm_data, hf_processor_mm_kwargs
hf_data, hf_kwargs
)
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**prepared_kwargs),
dict(text=prompt_text, **prepared_data),
prepared_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
videos = mm_data.get("videos")
if isinstance(videos, list) and videos:
if (
"videos" in hf_data
and isinstance(videos := hf_data["videos"], list)
and len(videos) > 0
):
# Videos are processed item-wise below; remove them from hf_data so
# the combined call at the end does not re-sample the raw frames.
del hf_data["videos"]
video_grid_thw_lst = []
pixel_values_videos_lst = []
frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor)
swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id
mm_data.pop("videos")
for item in videos:
video_array, metadata = item
# don't update mm_kwargs inplace
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
video_mm_kwargs = dict(**hf_kwargs)
video_mm_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", True
)
@@ -1667,9 +1674,9 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
swap_video_frame_tokens = False
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
self.info.get_hf_processor(**hf_kwargs),
dict(text=prompt_text, **hf_data),
hf_kwargs,
)
if swap_video_frame_tokens:
input_ids = processed_data["input_ids"]
@@ -1677,9 +1684,9 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
processed_data["input_ids"] = input_ids
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+13 -14
View File
@@ -44,6 +44,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.transformers_utils.processor import cached_processor_from_config
@@ -739,26 +740,24 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
chunk_counts.append(min(n_chunks, max_windows))
return chunk_counts
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
if "audios" in mm_data:
mm_data["audio"] = mm_data.pop("audios")
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
return hf_inputs._replace(
hf_kwargs=dict(
hf_inputs.hf_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
+5 -19
View File
@@ -110,7 +110,7 @@ class GraniteSpeechAudioInputs(TensorSchema):
"""List of audio embedding sizes for each item in batch."""
class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
class GraniteSpeechProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
feature_extractor = self.get_hf_processor().audio_processor
@@ -135,7 +135,7 @@ class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
### Input Processing & Multimodal utils
class GraniteSpeechMultiModalProcessor(
BaseMultiModalProcessor[GraniteSpeechMultiModalProcessingInfo]
BaseMultiModalProcessor[GraniteSpeechProcessingInfo]
):
def _get_mm_fields_config(
self,
@@ -181,23 +181,9 @@ class GraniteSpeechMultiModalProcessor(
)
]
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
# GraniteSpeechFeatureExtractor accepts "audio"
mm_data["audio"] = audios
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
@@ -216,7 +202,7 @@ class GraniteSpeechMultiModalProcessor(
class GraniteSpeechDummyInputsBuilder(
BaseDummyInputsBuilder[GraniteSpeechMultiModalProcessingInfo]
BaseDummyInputsBuilder[GraniteSpeechProcessingInfo]
):
def get_dummy_mm_data(
self,
@@ -565,7 +551,7 @@ class GraniteSpeechCTCEncoder(nn.Module):
@MULTIMODAL_REGISTRY.register_processor(
GraniteSpeechMultiModalProcessor,
info=GraniteSpeechMultiModalProcessingInfo,
info=GraniteSpeechProcessingInfo,
dummy_inputs=GraniteSpeechDummyInputsBuilder,
)
class GraniteSpeechForConditionalGeneration(
@@ -12,8 +12,8 @@ from .granite_speech import (
GraniteSpeechCTCEncoder,
GraniteSpeechDummyInputsBuilder,
GraniteSpeechForConditionalGeneration,
GraniteSpeechMultiModalProcessingInfo,
GraniteSpeechMultiModalProcessor,
GraniteSpeechProcessingInfo,
)
ISO639_1_SUPPORTED_LANGS = {
@@ -60,7 +60,7 @@ class GraniteSpeechPlusCTCEncoder(GraniteSpeechCTCEncoder):
@MULTIMODAL_REGISTRY.register_processor(
GraniteSpeechMultiModalProcessor,
info=GraniteSpeechMultiModalProcessingInfo,
info=GraniteSpeechProcessingInfo,
dummy_inputs=GraniteSpeechDummyInputsBuilder,
)
class GraniteSpeechPlusForConditionalGeneration(GraniteSpeechForConditionalGeneration):
+2 -2
View File
@@ -21,7 +21,7 @@ from vllm.multimodal.parse import (
MultiModalDataItems,
)
from vllm.multimodal.processing.processor import (
MultiModalProcessingInfo,
MultiModalProcessingResult,
ProcessorInputs,
PromptReplacement,
TimingContext,
@@ -139,7 +139,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
# The processor logic is different for len(images) <= 1 vs > 1
# Since the processing cache assumes that the processor output is
# invariant of how many images are passed per prompt, we only
@@ -261,44 +261,8 @@ class HCXVisionV2MultiModalProcessor(
):
"""Multimodal processor for HyperCLOVAX V2 (32B Think model)."""
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
images = mm_data.get("images")
videos = mm_data.get("videos")
# Get the HF processor
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
# Build data dict for HF processor (images/videos only)
# NOTE: We pass the prompt as-is without token normalization.
# Token expansion is handled by vLLM via _get_prompt_updates.
data: dict[str, object] = dict(
text=prompt_text,
images=images,
videos=videos,
)
processed_data = self.info.ctx.call_hf_processor(
hf_processor=hf_processor,
data=data,
kwargs=hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_prompt_updates(
self,
+21 -21
View File
@@ -52,6 +52,7 @@ from vllm.multimodal.processing import (
PromptUpdateDetails,
cached_encode,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -240,31 +241,32 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]):
def _apply_hf_processor_main(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
image_processor = self.info.get_hf_processor().image_processor
if getattr(image_processor, "backend", "pil") == "pil":
hf_processor_mm_kwargs = {
"input_data_format": "channels_last",
**hf_processor_mm_kwargs,
}
hf_inputs = hf_inputs._replace(
hf_kwargs={
"input_data_format": "channels_last",
**hf_inputs.hf_kwargs,
}
)
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
hf_processor_mm_kwargs,
)
return hf_inputs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
images = mm_data.get("images", [])
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
@@ -289,8 +291,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
processed_data["pixel_values"].squeeze_(0)
processed_data["pixel_attention_mask"].squeeze_(0)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
+20 -18
View File
@@ -335,28 +335,30 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]):
"""Basic image-only MultiModalProcessor for InternS1-style models."""
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
mm_data = dict(mm_data)
videos = mm_data.pop("videos", [])
images = mm_data.pop("images", [])
videos = hf_data.pop("videos", [])
images = hf_data.pop("images", [])
assert isinstance(videos, list)
assert isinstance(images, list)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_kwargs)
tokenizer = hf_processor.tokenizer
vocab = tokenizer.get_vocab()
@@ -374,9 +376,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
image_pixel_values = []
for image in images:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
self.info.get_hf_processor(**hf_kwargs),
dict(text=hf_processor.image_token, **{"images": image}),
hf_processor_mm_kwargs,
hf_kwargs,
)
image_pixel_values.append(processed_data.pop("pixel_values"))
@@ -398,9 +400,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
video_pixel_values = []
for video in videos:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
self.info.get_hf_processor(**hf_kwargs),
dict(text=hf_processor.video_token, **{"videos": video}),
hf_processor_mm_kwargs,
hf_kwargs,
)
video_pixel_values.append(processed_data.pop("pixel_values"))
@@ -430,9 +432,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
processed_data.update(text_outputs)
processed_data.update(image_outputs)
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+8 -7
View File
@@ -13,7 +13,7 @@ from vllm.model_executor.layers.pooler import DispatchPooler
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalKwargsItems
from vllm.multimodal.processing.processor import (
MultiModalProcessingInfo,
MultiModalProcessingResult,
ProcessorInputs,
TimingContext,
)
@@ -63,19 +63,19 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
mm_info = super()._cached_apply_hf_processor(inputs, timing_ctx)
) -> MultiModalProcessingResult:
mm_res = super()._cached_apply_hf_processor(inputs, timing_ctx)
# Score inputs are query-first, while the prompt template is document-first.
mm_kwargs = MultiModalKwargsItems(
{
modality: list(reversed(items))
for modality, items in mm_info.kwargs.items()
for modality, items in mm_res.kwargs.items()
}
)
mm_hashes = {
modality: list(reversed(hashes))
for modality, hashes in mm_info.hashes.items()
for modality, hashes in mm_res.hashes.items()
}
mm_prompt_updates = {
modality: [
@@ -85,10 +85,11 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
]
for item_idx, updates in enumerate(reversed(item_updates))
]
for modality, item_updates in mm_info.prompt_updates.items()
for modality, item_updates in mm_res.prompt_updates.items()
}
return MultiModalProcessingInfo(
return MultiModalProcessingResult(
prompt_ids=mm_res.prompt_ids,
kwargs=mm_kwargs,
hashes=mm_hashes,
prompt_updates=mm_prompt_updates,
+14 -10
View File
@@ -459,24 +459,27 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
def media_token_id(self) -> int:
return self.info.get_hf_config().text_config.eos_token_id + 1
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Run the underlying HF processor on text and image data."""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data or not mm_data.get("images", []):
return BatchFeature(dict(passthrough_data))
if not hf_data or not hf_data.get("images", []):
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
# Images
image_inputs = mm_data.get("images", [])
image_inputs = hf_data.get("images", [])
pixel_sizes = []
if not isinstance(image_inputs[0], Image.Image):
image_inputs = [Image.fromarray(image) for image in image_inputs]
@@ -530,8 +533,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
pixel_sizes=torch.tensor(pixel_sizes),
)
processed_data = BatchFeature(combined_outputs, tensor_type="pt")
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_prompt_updates(
self,
+1 -1
View File
@@ -1160,7 +1160,7 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):
class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_prompt_updates(
+1 -1
View File
@@ -373,7 +373,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_prompt_updates(
+8 -24
View File
@@ -47,6 +47,7 @@ from vllm.multimodal.processing import (
)
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
HFMultiModalInputs,
ProcessorInputs,
)
from vllm.sequence import IntermediateTensors
@@ -227,27 +228,17 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser):
class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]):
"""vLLM multi-modal processor wrapper for Kimi-Audio."""
def _apply_hf_processor_main(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Call the HuggingFace processor."""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
# Convert mm_data format: {'audios': [...]} -> {'audio': ...}
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# Convert audio format: [(array, sr), ...] -> [array, ...]
# KimiAudioProcessor expects raw numpy arrays
if audios:
mm_data = hf_inputs.hf_data
if audios := mm_data.pop("audio", []):
audio_arrays = []
for aud in audios:
if isinstance(aud, (tuple, list)) and len(aud) == 2:
@@ -259,14 +250,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
audio_arrays.append(aud)
mm_data["audio"] = audio_arrays
# Use the context's call_hf_processor for proper handling
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
return hf_inputs
def _get_mm_fields_config(
self,
+1 -1
View File
@@ -236,7 +236,7 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_mm_fields_config(
+1 -1
View File
@@ -406,7 +406,7 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):
class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+2 -26
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, Sequence
from typing import TypeVar
import torch
import torch.nn as nn
@@ -35,8 +34,6 @@ from vllm.multimodal.processing import (
PromptUpdateDetails,
)
_I = TypeVar("_I", bound=Mistral3ProcessingInfo)
class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
def get_vision_encoder_info(
@@ -49,8 +46,8 @@ class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
return PixtralHFEncoderInfo(self.get_hf_config())
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
@@ -62,27 +59,6 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
if not mm_data:
return processed_data
input_ids = processed_data.get("input_ids")
if input_ids is not None:
processor = self.info.get_hf_processor()
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
break_id = vocab.get(processor.image_break_token)
end_id = vocab.get(processor.image_end_token)
# create mask to remove break/end tokens
keep_mask = ~torch.isin(
input_ids,
torch.tensor([break_id, end_id]),
)
processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0)
if "attention_mask" in processed_data:
processed_data["attention_mask"] = processed_data["attention_mask"][
keep_mask
].unsqueeze(0)
# un-pad pixel_values per-image so caches remain independent.
pixel_values = processed_data.get("pixel_values")
if pixel_values is not None:
+1 -1
View File
@@ -320,7 +320,7 @@ class PixtralHFProcessingInfo(BaseLlavaProcessingInfo):
class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+1 -1
View File
@@ -209,7 +209,7 @@ class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]):
class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_mm_fields_config(
+25 -21
View File
@@ -324,33 +324,36 @@ class LlavaOnevisionMultiModalProcessor(
pixel_values_videos=MultiModalFieldConfig.batched("video"),
)
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
mm_data = dict(mm_data)
videos = mm_data.pop("videos", [])
videos = hf_data.pop("videos", [])
assert isinstance(videos, list)
if not videos:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
self.info.get_hf_processor(**hf_kwargs),
dict(text=prompt_text, **hf_data),
hf_kwargs,
)
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
processed_data.update(passthrough_data)
return processed_data
# LLaVA-OneVision processor doesn't support multiple videos
# with different sizes when converting back to tensors
@@ -360,13 +363,13 @@ class LlavaOnevisionMultiModalProcessor(
image_token = processor.image_token
video_token = processor.video_token
images = mm_data.pop("images", [])
images = hf_data.pop("images", [])
assert isinstance(images, list)
if images:
processor_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
self.info.get_hf_processor(**hf_kwargs),
dict(text=image_token * len(images), **{"images": images}),
hf_processor_mm_kwargs,
hf_kwargs,
)
image_outputs = {
k: v
@@ -379,9 +382,9 @@ class LlavaOnevisionMultiModalProcessor(
pixel_values_videos = []
for video in videos:
item_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
self.info.get_hf_processor(**hf_kwargs),
dict(text=video_token, **{"videos": video}),
hf_processor_mm_kwargs,
hf_kwargs,
)
pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
@@ -394,8 +397,9 @@ class LlavaOnevisionMultiModalProcessor(
**video_outputs,
)
processed_data = BatchFeature(combined_outputs)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_prompt_updates(
self,
+37 -33
View File
@@ -1556,23 +1556,26 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
class LlavaOnevision2MultiModalProcessor(
BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]
):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_kwargs)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_kwargs)
merged_kwargs.setdefault("return_tensors", "pt")
call_kwargs = {
k: v
@@ -1589,17 +1592,17 @@ class LlavaOnevision2MultiModalProcessor(
"codec_config",
}
}
mm_data = dict(mm_data)
# Explicit None + length checks: ``mm_data[...]`` may be a list, numpy
# array, or tensor, and ``and <array>`` would raise on the ambiguous
# truth value of a multi-element array.
_videos = mm_data.get("videos")
_videos = hf_data.get("videos")
if _videos is not None and not isinstance(_videos, Sized):
raise TypeError(f"object of type '{type(_videos).__name__}' has no len()")
videos_present = _videos is not None and len(_videos) > 0
codec_video_paths = (
_extract_codec_video_paths(mm_data["videos"]) if videos_present else None
_extract_codec_video_paths(hf_data["videos"]) if videos_present else None
)
is_codec_marker = codec_video_paths is not None
# Fallback: caller passed video_backend=codec via mm_processor_kwargs
@@ -1613,7 +1616,7 @@ class LlavaOnevision2MultiModalProcessor(
and call_kwargs.get("video_backend") == "codec"
)
if is_codec_kwarg:
raw = mm_data["videos"]
raw = hf_data["videos"]
if isinstance(raw, str):
codec_video_paths = [raw]
elif isinstance(raw, (list, tuple)) and all(
@@ -1638,7 +1641,7 @@ class LlavaOnevision2MultiModalProcessor(
# Codec backend: HF processor consumes the path string directly and
# performs decode + canvas-packing internally. The dummy ndarray
# we attached during prepare_codec_video_input is discarded here.
mm_data["videos"] = (
hf_data["videos"] = (
codec_video_paths
if len(codec_video_paths) > 1
else codec_video_paths[0]
@@ -1648,11 +1651,9 @@ class LlavaOnevision2MultiModalProcessor(
# ``video_backend="codec"`` via mm_kwargs so the wrapped processor
# dispatches to its codec branch.
output = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(
**{**hf_processor_mm_kwargs, "video_backend": "codec"}
),
dict(text=prompt_text, **mm_data),
{**hf_processor_mm_kwargs, "video_backend": "codec"},
self.info.get_hf_processor(**{**hf_kwargs, "video_backend": "codec"}),
dict(text=prompt_text, **hf_data),
{**hf_kwargs, "video_backend": "codec"},
)
data = dict(output)
processed_data = BatchFeature(
@@ -1660,8 +1661,9 @@ class LlavaOnevision2MultiModalProcessor(
data, codec_video_paths, hf_processor
)
)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
# ---- Frame backend (registered LlavaOnevision2VideoBackend) ------
# Every non-codec video reaches here as a ``(frames_ndarray, metadata)``
@@ -1676,7 +1678,7 @@ class LlavaOnevision2MultiModalProcessor(
# provided by the backend's ``compute_frames_index_to_sample``; SSRF /
# local-file gating is enforced by the connector before decoding.
if videos_present:
timestamp_decimals = hf_processor_mm_kwargs.get(
timestamp_decimals = hf_kwargs.get(
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
)
if not isinstance(
@@ -1690,7 +1692,7 @@ class LlavaOnevision2MultiModalProcessor(
per_video_frames: list[list[Image.Image]] = []
per_video_timestamps: list[list[float]] = []
videos = mm_data["videos"]
videos = hf_data["videos"]
if not isinstance(videos, Iterable):
raise TypeError(f"Unsupported videos input: {type(videos)}")
for item in videos:
@@ -1714,7 +1716,7 @@ class LlavaOnevision2MultiModalProcessor(
# order) -- otherwise mixed image+video requests bind frames to the
# wrong placeholder. ``row_is_video`` labels each grid row so outputs
# can be split back into per-modality keys below.
merged_mm_data = dict(mm_data)
merged_mm_data = dict(hf_data)
existing_images = merged_mm_data.pop("images", None)
caller_images: list[Image.Image] = []
if existing_images:
@@ -1760,9 +1762,9 @@ class LlavaOnevision2MultiModalProcessor(
# **kwargs to the image processor, so passing the full merged kwarg
# set here is a no-op beyond return_tensors/padding.
output = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
self.info.get_hf_processor(**hf_kwargs),
dict(text=new_prompt, **merged_mm_data),
hf_processor_mm_kwargs,
hf_kwargs,
)
data = dict(output)
@@ -1820,20 +1822,22 @@ class LlavaOnevision2MultiModalProcessor(
)
processed_data = BatchFeature(data)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
# ---- Image-only / text-only call --------------------------------
# No videos present: delegate to ``ctx.call_hf_processor``, which
# runs the wrapped processor over the (possibly empty) image set and
# applies float-tensor dtype postprocessing automatically.
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
self.info.get_hf_processor(**hf_kwargs),
dict(text=prompt_text, **hf_data),
hf_kwargs,
)
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
processed_data.update(passthrough_data)
return processed_data
def _rename_codec_outputs_to_video(
self,
+22 -23
View File
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.midashenglm import DashengConfig
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -582,35 +583,33 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
class MiDashengLMMultiModalProcessor(
BaseMultiModalProcessor[MiDashengLMProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
assert isinstance(audios, list)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
min_audio_len = self.info.get_min_audio_len()
processed_audios = [
np.pad(
audio,
(0, min_audio_len - audio.shape[-1]),
mode="constant",
constant_values=0,
)
if isinstance(audio, np.ndarray) and audio.shape[-1] < min_audio_len
else audio
for audio in audios
]
if audios := hf_inputs.hf_data.get("audio"):
assert isinstance(audios, list)
if processed_audios:
mm_data["audio"] = processed_audios
min_audio_len = self.info.get_min_audio_len()
hf_inputs.hf_data["audio"] = [
np.pad(
audio,
(0, min_audio_len - audio.shape[-1]),
mode="constant",
constant_values=0,
)
if isinstance(audio, np.ndarray) and audio.shape[-1] < min_audio_len
else audio
for audio in audios
]
return mm_data, hf_processor_mm_kwargs
return hf_inputs
def _get_mm_fields_config(
self,
+10 -28
View File
@@ -46,6 +46,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.transformers_utils.configs.mimo_v2_omni import Mimo_VLVisionConfig
from vllm.transformers_utils.processors.mimo_v2_omni import (
MiMoOmniProcessor,
@@ -930,27 +931,14 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")
return fields
def _apply_hf_processor_main(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.
Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses
the singular form.
"""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
# Remap audios → audio (MiMoOmniProcessor uses singular param name)
if "audios" in mm_data:
mm_data = {**mm_data, "audio": mm_data["audios"]}
mm_data = {k: v for k, v in mm_data.items() if k != "audios"}
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo."""
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_data = hf_inputs.hf_data
# Handle video_audio items: convert video part to (TCHW, timestamps) tuple
if "video_audio" in mm_data:
@@ -992,7 +980,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
audio=va_item.audio,
)
)
mm_data = {**mm_data, "video_audio": va_converted}
mm_data["video_audio"] = va_converted
if "videos" in mm_data:
converted: list[tuple[torch.Tensor, torch.Tensor]] = []
@@ -1026,15 +1014,9 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
timestamps = torch.arange(T, dtype=torch.float32) / self._INPUT_FPS
converted.append((frames, timestamps))
mm_data = {**mm_data, "videos": converted}
mm_data["videos"] = converted
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
return hf_inputs
def _get_prompt_updates(
self,
+1 -1
View File
@@ -445,7 +445,7 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
if (audios := mm_data.get("audios")) is None:
if (audios := mm_data.get("audio")) is None:
return {}
mm_items = self.info.parse_mm_data({"audio": audios}, validate=False)
+7 -7
View File
@@ -1059,19 +1059,18 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
tokenizer = self.info.get_tokenizer()
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
mm_inputs = self.process_mm_inputs(mm_data, hf_kwargs)
processed_data = BatchFeature(
{
@@ -1079,8 +1078,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
**mm_inputs,
}
)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
mm_data, hf_kwargs, passthrough_data, processed_data
)
def _get_prompt_updates(
self,
+2 -2
View File
@@ -248,7 +248,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
return ImageSize(width=width, height=height)
class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo]):
class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_images = mm_counts.get("image", 0)
@@ -280,7 +280,7 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo])
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+1 -1
View File
@@ -589,7 +589,7 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+21 -20
View File
@@ -1122,6 +1122,9 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
processor = self.info.get_hf_processor()
@@ -1140,28 +1143,28 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
dict(tokens=tokens),
)["input_ids"].tolist()
def _apply_hf_processor_main(
def _call_hf_processor(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_counts = mm_items.get_all_counts()
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not processor_data:
return BatchFeature(dict(passthrough_data))
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
processed_data = self.info.ctx.call_hf_processor(
hf_processor = self.info.get_hf_processor(**hf_kwargs)
return self.info.ctx.call_hf_processor(
hf_processor.process,
dict(text=prompt_text, **processor_data),
hf_processor_mm_kwargs,
hf_data,
hf_kwargs,
)
def _postprocess_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not hf_data:
return processed_data
hf_processor = self.info.get_hf_processor(**hf_kwargs)
tokenizer = hf_processor.tokenizer
image_patch_id = tokenizer.vocab[IMAGE_PATCH_TOKEN]
@@ -1169,7 +1172,7 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
processed_data.pop("input_ids")
if (images := processor_data.get("images")) is not None:
if (images := hf_data.get("images")) is not None:
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [
@@ -1193,8 +1196,6 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
processed_data["num_crops"] = num_crops
processed_data["img_patch_id"] = image_patch_id
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
+23 -18
View File
@@ -1932,6 +1932,9 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
processor = self.info.get_hf_processor()
tokenizer = processor.tokenizer
@@ -1946,19 +1949,20 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_counts = mm_items.get_all_counts()
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
mm_data = dict(processor_data)
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
hf_config = self.info.get_hf_config()
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_kwargs)
def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature:
res = hf_processor(text=text, images=images, videos=videos, **kwargs)
@@ -1973,7 +1977,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
tokenizer = hf_processor.tokenizer
image_processor = hf_processor.image_processor
if videos := mm_data.pop("videos", []):
if videos := hf_data.pop("videos", []):
assert isinstance(videos, Sequence)
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
@@ -1993,16 +1997,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
# NOTE: metadata.frames_indices indicates
# the sampled frames indices of pre-sampled videos, which is
# used to calculate the timestamps. Make sure that
# do_sample_frames in hf_processor_mm_kwargs is false for
# do_sample_frames in hf_kwargs is false for
# presampled videos.
# NOTE: a copy of hf_processor_mm_kwargs is created to update
# NOTE: a copy of hf_kwargs is created to update
# do_sample_frames, otherwise mm_hash for the object will be
# incorrect.
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
video_mm_kwargs = dict(**hf_kwargs)
if "do_sample_frames" not in video_mm_kwargs:
# molmo_utils already has "do_sample_frames" in
# hf_processor_mm_kwargs, don't overwrite it.
# hf_kwargs, don't overwrite it.
video_mm_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", False
)
@@ -2067,11 +2071,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
processed_data = self.info.ctx.call_hf_processor(
patched_call,
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
dict(text=prompt_text, **hf_data),
hf_kwargs,
)
if (images := mm_data.get("images")) is not None:
if (images := hf_data.get("images")) is not None:
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [
@@ -2113,10 +2117,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
)
processed_data.update(all_video_outputs)
processed_data.update(passthrough_data)
processed_data.pop("input_ids")
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+6 -21
View File
@@ -1356,32 +1356,17 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo
class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]):
def _apply_hf_processor_main(
def _call_hf_processor(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
mm_data["audio"] = audios
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
processor_kwargs = _filter_moss_audio_processor_config(hf_processor_mm_kwargs)
processed_data = self.info.ctx.call_hf_processor(
processor_kwargs = _filter_moss_audio_processor_config(dict(hf_kwargs))
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**processor_kwargs),
mm_data,
hf_data,
{},
)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
@@ -66,7 +66,7 @@ from vllm.multimodal.processing import (
PromptUpdateDetails,
cached_encode,
)
from vllm.multimodal.processing.processor import ProcessorInputs
from vllm.multimodal.processing.processor import HFMultiModalInputs, ProcessorInputs
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -465,17 +465,19 @@ class MossTranscribeDiarizeDummyInputsBuilder(
class MossTranscribeDiarizeMultiModalProcessor(
BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
audios = _get_audios_from_mm_data(mm_data)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
if hf_inputs.hf_data:
hf_inputs.hf_data["audio"] = _get_audios_from_mm_data(hf_inputs.hf_data)
return dict(audio=audios), hf_processor_mm_kwargs
return hf_inputs
def _postprocess_hf_mm_data(
self,
+10 -7
View File
@@ -275,16 +275,18 @@ class MuseGlimmerMultiModalProcessor(
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
processor = self.info.get_hf_processor(**hf_kwargs)
tokenizer = processor.tokenizer
config = self.info.get_hf_config()
images = mm_data.get("images", ())
@@ -371,8 +373,9 @@ class MuseGlimmerMultiModalProcessor(
video_feature_sizes=torch.tensor(video_sizes),
)
processed_data = BatchFeature(data=data, tensor_type=None)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
mm_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+23 -8
View File
@@ -70,6 +70,7 @@ from vllm.multimodal.processing import (
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
HFMultiModalInputs,
PromptReplacement,
PromptUpdate,
cached_encode,
@@ -196,6 +197,9 @@ NanoNemotronVLVideoInputs: TypeAlias = (
class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor:
# Not accepted by NanoNemotronVLProcessor.__init__
kwargs.pop("truncation", None)
return self.ctx.init_processor(
NanoNemotronVLProcessor,
config=self.get_hf_config(),
@@ -358,9 +362,23 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
class NanoNemotronVLMultiModalProcessor(
BaseMultiModalProcessor[NanoNemotronVLProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# NanoNemotronVLProcessor expects "audios" instead of "audio"
hf_data = hf_inputs.hf_data
if "audio" in hf_data:
hf_data["audios"] = hf_data.pop("audio")
return hf_inputs
def _get_image_fields_config(self, hf_inputs: BatchFeature):
if self.info.is_dynamic_tiler:
pixel_values_flat = MultiModalFieldConfig.batched("image")
@@ -730,15 +748,12 @@ class NanoNemotronVLMultiModalProcessor(
# Bypass the cached path: the HF processor must receive the
# prompt (with injected <so_embedding>) and the audio data
# together so it can perform audio-token replacement natively.
mm_info = self._apply_hf_processor(inputs, timing_ctx)
prompt_ids = self._postprocess_prompt(inputs.prompt)
mm_res = self._apply_hf_processor(inputs, timing_ctx)
with timing_ctx.record("apply_prompt_updates"):
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
mm_items=mm_items,
prompt_ids=prompt_ids,
mm_kwargs=mm_info.kwargs,
mm_prompt_updates=mm_info.prompt_updates,
mm_res=mm_res,
)
mm_placeholder_ranges = {
@@ -749,8 +764,8 @@ class NanoNemotronVLMultiModalProcessor(
return MultiModalInput(
type="multimodal",
prompt_token_ids=prompt_ids,
mm_kwargs=mm_info.kwargs,
mm_hashes=mm_info.hashes,
mm_kwargs=mm_res.kwargs,
mm_hashes=mm_res.hashes,
mm_placeholders=mm_placeholder_ranges,
)
+3 -3
View File
@@ -326,6 +326,9 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]):
class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def image_indicators_to_visual_tokens(
self,
image_indicators: list[int],
@@ -340,9 +343,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
# -300 is image_atom token, filter them out
return [vte_vocab_size + x + 300 for x in image_indicators if x < -300]
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
+1 -1
View File
@@ -327,7 +327,7 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
if x >= INDICATOR_IDS[0]
]
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
+20 -19
View File
@@ -243,36 +243,37 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing
class PaddleOCRVLMultiModalProcessor(
BaseMultiModalProcessor[PaddleOCRVLProcessingInfo]
):
def _apply_hf_processor_main(
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _call_hf_processor(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
final_mm_kwargs = dict(hf_processor_mm_kwargs or {})
final_mm_kwargs = dict(hf_kwargs or {})
final_mm_kwargs.setdefault("images_kwargs", {})
# vLLM use PIL.Image, always set channel_last
final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
processed_data = self.info.ctx.call_hf_processor(
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**final_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
hf_data,
hf_kwargs,
)
def _postprocess_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not hf_data:
return processed_data
num_patches_per_image = processed_data["image_grid_thw"].prod(-1)
processed_data["pixel_values"] = processed_data["pixel_values"].split(
num_patches_per_image.tolist()
)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
+1 -20
View File
@@ -390,28 +390,9 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
input_ids = processed_data["input_ids"]
assert isinstance(input_ids, torch.Tensor)
# Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
# which will cause OverflowError when decoding the prompt_ids.
# Therefore, we need to do an early replacement here
input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID)
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+38 -34
View File
@@ -45,6 +45,7 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
HFMultiModalInputs,
PromptReplacement,
PromptUpdate,
ResolvedPromptUpdate,
@@ -867,40 +868,38 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
def _apply_hf_processor_main(
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# The Phi4MM processor expects "audios" instead of "audio"
hf_data = hf_inputs.hf_data
if "audio" in hf_data:
hf_data["audios"] = hf_data.pop("audio")
sr = self.info.get_feature_extractor(**hf_inputs.hf_kwargs).sampling_rate
if audio_data := hf_data.get("audios", []):
assert isinstance(audio_data, Sequence)
hf_data["audios"] = [(data, sr) for data in audio_data]
return hf_inputs
def _postprocess_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not hf_data:
return processed_data
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
sr = self.info.get_feature_extractor(**hf_processor_mm_kwargs).sampling_rate
raw_audio_data = mm_data.get("audios", [])
assert isinstance(raw_audio_data, Sequence)
audio_data = []
for data in raw_audio_data:
assert isinstance(data, (list, np.ndarray, torch.Tensor))
audio_data.append(data)
processor_data = dict(text=prompt_text, **mm_data)
if audio_data:
processor_data["audios"] = [(data, sr) for data in audio_data]
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
processor_data,
hf_processor_mm_kwargs,
)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_kwargs)
num_img_tokens = [
self.info.get_num_image_tokens(
image_width=img_size[0],
@@ -911,21 +910,26 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
]
processed_data["num_img_tokens"] = num_img_tokens
audio_features = processed_data["input_audio_embeds"]
audios = hf_data.get("audios", [])
assert isinstance(audios, list)
audio_features = processed_data.get("input_audio_embeds")
if audio_features is None:
return processed_data
sr = self.info.get_feature_extractor(**hf_kwargs).sampling_rate
feature_sizes = [
self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data
self.info.get_audio_num_frames(len(audio), sr) for audio, _ in audios
]
processed_data["input_audio_embeds"] = [
audio_features[idx, :size] for idx, size in enumerate(feature_sizes)
]
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
return dict(
input_image_embeds=MultiModalFieldConfig.batched("image"),
+2 -33
View File
@@ -143,39 +143,8 @@ class Phi4SiglipDummyInputsBuilder(
class Phi4SiglipMultiModalProcessor(
BaseMultiModalProcessor[Phi4SiglipProcessingInfo],
):
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
processed = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
# The HF processor's tokenizer_image_token() replaces the "<image>"
# string with IMAGE_TOKEN_INDEX (-200) in input_ids. This breaks
# vLLM's prompt-replacement pipeline which needs to find "<image>"
# as normal sub-tokens. Re-tokenize with the plain tokenizer so
# that "<image>" stays as sub-tokens and can be located by
# PromptReplacement.
# NOTE: tokenizer.__call__() (not .encode()) must be used so that
# added/special tokens like <|user|>, <|end|> are kept as single IDs.
tokenizer = self.info.get_tokenizer()
new_ids = tokenizer(prompt_text).input_ids
processed["input_ids"] = torch.tensor([new_ids])
processed_data = processed
processed_data.update(passthrough_data)
return processed_data
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_mm_fields_config(
self,
+17 -38
View File
@@ -14,7 +14,7 @@ from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
from mistral_common.protocol.instruct.messages import UserMessage
from mistral_common.protocol.instruct.request import ChatCompletionRequest
from packaging.version import Version
from transformers import BatchFeature, PixtralVisionConfig
from transformers import PixtralVisionConfig
from transformers.models.pixtral.image_processing_pixtral import (
_num_image_tokens as _get_pixtral_hf_num_image_tokens,
)
@@ -41,7 +41,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsOptionalItems,
NestedTensors,
)
from vllm.multimodal.parse import (
@@ -53,7 +52,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
MultiModalPromptUpdates,
HFMultiModalInputs,
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
ProcessorInputs,
PromptReplacement,
@@ -67,7 +67,6 @@ from vllm.transformers_utils.processors.pixtral import (
MistralCommonImageProcessor,
MistralCommonPixtralProcessor,
)
from vllm.utils.collection_utils import is_list_of
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -268,21 +267,19 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsOptionalItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_item_counts = mm_items.get_all_counts()
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
mm_placeholders = self._find_mm_placeholders(
prompt_ids,
mm_prompt_updates,
mm_res.prompt_ids,
mm_res.prompt_updates,
)
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
return prompt_ids, mm_placeholders
return mm_res.prompt_ids, mm_placeholders
def _get_mm_fields_config(
self,
@@ -291,35 +288,17 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
) -> Mapping[str, MultiModalFieldConfig]:
return dict(images=MultiModalFieldConfig.batched("image"))
def _apply_hf_processor_main(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# Avoid padding issue
return hf_inputs._replace(
hf_kwargs=dict(hf_inputs.hf_kwargs, return_tensors=None)
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
# Avoid padding issue
dict(**hf_processor_mm_kwargs, return_tensors=None),
)
# Missing batch dimension
if is_list_of(outputs["input_ids"], int):
outputs["input_ids"] = [outputs["input_ids"]]
processed_data = outputs
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates(
self,
@@ -82,6 +82,8 @@ from vllm.multimodal.processing import (
)
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
HFMultiModalInputs,
MultiModalProcessingResult,
MultiModalPromptUpdates,
PlaceholderFeaturesInfo,
PromptReplacement,
@@ -498,6 +500,64 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
class Qwen2_5OmniThinkerMultiModalProcessor(
BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_counts = mm_items.get_all_counts()
if hf_kwargs.get("use_audio_in_video"):
num_videos = mm_counts.get("video", 0)
if mm_counts.get("audio", 0) < num_videos:
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
if num_videos and "text" in hf_inputs.hf_data:
standalone_mm_counts = dict(mm_counts)
standalone_mm_counts["audio"] -= num_videos
hf_inputs.hf_data["text"] = self._get_hf_mm_text(standalone_mm_counts)
hf_data = hf_inputs.hf_data
normalized_kwargs = hf_inputs.hf_kwargs
audios = hf_data.pop("audio", [])
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
if audios:
hf_data["audio"] = audios
elif (
normalized_kwargs.get("use_audio_in_video")
and "audio" not in hf_data
and hf_data.get("videos")
):
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
merged = self.info.ctx.get_merged_mm_kwargs(normalized_kwargs)
if hf_data.get("videos") and (
merged.keys() & {"size", "min_pixels", "max_pixels"}
):
normalized_kwargs = dict(normalized_kwargs)
video_size = dict(self.info.get_hf_processor().video_processor.size)
size_override = merged.get("size")
if size_override is not None:
video_size = video_size | size_override
min_pixels = merged.get("min_pixels")
if min_pixels is not None:
video_size["shortest_edge"] = min_pixels
max_pixels = merged.get("max_pixels")
if max_pixels is not None:
video_size["longest_edge"] = max_pixels
normalized_kwargs["size"] = video_size
return hf_inputs._replace(hf_kwargs=normalized_kwargs)
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -570,12 +630,14 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`."""
mm_item_counts = mm_items.get_all_counts()
prompt_ids = mm_res.prompt_ids
mm_kwargs = mm_res.kwargs
mm_prompt_updates = mm_res.prompt_updates
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
@@ -789,101 +851,33 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
),
]
def _apply_hf_processor_main(
def _postprocess_hf_mm_data(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
"""Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`."""
mm_counts = mm_items.get_all_counts()
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
if use_audio_in_video and "video" in mm_counts:
if mm_counts.get("audio", 0) < mm_counts["video"]:
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
mm_counts["audio"] -= mm_counts["video"]
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
mm_data = dict(processor_data)
audios = mm_data.pop("audios", [])
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
if audios:
# NOTE: Qwen2.5-Omni processor accept "audio"
mm_data["audio"] = audios
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
)
elif (
hf_processor_mm_kwargs.get("use_audio_in_video")
and "audio" not in mm_data
and mm_data.get("videos")
):
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
# and populated mm_data["audio"] itself before delegating here,
# in which case this isn't a genuine "no audio" case. Likewise,
# mm_data can be empty on a multimodal-processor-cache hit, where
# there's nothing to (re-)process this call and the real result
# comes from the cache — not a genuine "no audio" case either.
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
if mm_data.get("videos") and (
merged.keys() & {"size", "min_pixels", "max_pixels"}
):
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
video_size = dict(self.info.get_hf_processor().video_processor.size)
size_override = merged.get("size")
if size_override is not None:
video_size = video_size | size_override
min_pixels = merged.get("min_pixels")
if min_pixels is not None:
video_size["shortest_edge"] = min_pixels
max_pixels = merged.get("max_pixels")
if max_pixels is not None:
video_size["longest_edge"] = max_pixels
hf_processor_mm_kwargs["size"] = video_size
hf_inputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
input_features = hf_inputs.pop("input_features", None)
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
if "input_audio_features" not in hf_inputs and input_features is not None:
input_features = processed_data.pop("input_features", None)
feature_attention_mask = processed_data.get("feature_attention_mask", None)
if "input_audio_features" not in processed_data and input_features is not None:
if feature_attention_mask is not None:
input_features = input_features.permute(0, 2, 1)[
feature_attention_mask.bool()
].permute(1, 0)
hf_inputs["input_audio_features"] = input_features
processed_data["input_audio_features"] = input_features
if (
"audio_feature_lengths" not in hf_inputs
"audio_feature_lengths" not in processed_data
and feature_attention_mask is not None
):
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
processed_data["audio_feature_lengths"] = feature_attention_mask.sum(-1)
video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
video_second_per_grid = processed_data.get("video_second_per_grid", None)
if video_second_per_grid is not None:
hf_inputs["second_per_grid_ts"] = video_second_per_grid
processed_data["second_per_grid_ts"] = video_second_per_grid
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
processed_data = hf_inputs
processed_data.update(passthrough_data)
processed_data.pop("input_ids")
use_audio_in_video = hf_kwargs.get("use_audio_in_video", False)
processed_data["use_audio_in_video"] = torch.tensor(use_audio_in_video)
processed_data.pop("input_ids", None)
return processed_data
+14 -17
View File
@@ -59,6 +59,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -233,29 +234,25 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
mm_data["audio"] = audios
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
truncation=True,
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
return hf_inputs._replace(
hf_kwargs=dict(
hf_inputs.hf_kwargs,
sampling_rate=feature_extractor.sampling_rate,
truncation=True,
)
)
return mm_data, hf_processor_mm_kwargs
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -37,11 +37,10 @@ from vllm.model_executor.models.qwen3_asr import (
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
from vllm.multimodal.inputs import MultiModalKwargsOptionalItems
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing.processor import (
BaseDummyInputsBuilder,
MultiModalPromptUpdates,
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
cached_encode,
)
@@ -114,10 +113,10 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsOptionalItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_kwargs = mm_res.kwargs
audios = mm_kwargs.get("audio", [])
assert len(audios) == 1, (
f"Expected only one audio input for realtime, got {len(audios)}"
@@ -147,7 +146,7 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
# Find the audio_pad token position and expand it to audio_len tokens
expanded_ids = list[int]()
pad_start_idx = -1
for i, tid in enumerate(prompt_ids):
for i, tid in enumerate(mm_res.prompt_ids):
if tid == audio_pad_id and pad_start_idx == -1:
pad_start_idx = i
expanded_ids.extend([audio_pad_id] * audio_len)
@@ -73,6 +73,8 @@ from vllm.multimodal.inputs import (
)
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems
from vllm.multimodal.processing.processor import (
HFMultiModalInputs,
MultiModalProcessingResult,
MultiModalPromptUpdates,
PlaceholderFeaturesInfo,
PromptReplacement,
@@ -1219,13 +1221,26 @@ Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder
class Qwen3OmniMoeThinkerMultiModalProcessor(
Qwen2_5OmniThinkerMultiModalProcessor,
):
def _get_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
audios = processor_data.get("audios")
if mm_items.get_count("audio", strict=False):
# TODO(Isotr0py): Remove this patch after upstream fix PR
# released and Transformers version update:
# https://github.com/huggingface/transformers/pull/41473
hf_inputs = hf_inputs._replace(
hf_kwargs={
**hf_inputs.hf_kwargs,
"audio_kwargs": dict(hf_inputs.hf_kwargs.get("audio_kwargs") or {}),
"text_kwargs": dict(hf_inputs.hf_kwargs.get("text_kwargs") or {}),
}
)
audios = hf_inputs.hf_data.get("audio")
if audios:
feature_extractor = self.info.get_feature_extractor()
hop_length = feature_extractor.hop_length
@@ -1239,47 +1254,30 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
# To make sure the cache works with padding=True, we pre-padded
# the audio to a multiple of hop_length.
processor_data = dict(processor_data)
processor_data["audios"] = [
hf_inputs.hf_data["audio"] = [
pad_to_hop_length(audio)
if isinstance(audio, np.ndarray)
else (pad_to_hop_length(audio[0]), audio[1])
for audio in audios
]
return processor_data, passthrough_data
return hf_inputs
def _apply_hf_processor_main(
def _postprocess_hf_mm_data(
self,
mm_items: MultiModalDataItems,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if mm_items.get_all_counts().get("audio", 0):
# TODO(Isotr0py): Remove this patch after upstream fix PR
# released and Transformers version update:
# https://github.com/huggingface/transformers/pull/41473
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
hf_processor_mm_kwargs["audio_kwargs"] = dict(
hf_processor_mm_kwargs.get("audio_kwargs") or {}
)
hf_processor_mm_kwargs["text_kwargs"] = dict(
hf_processor_mm_kwargs.get("text_kwargs") or {}
)
processed_data = super()._apply_hf_processor_main(
mm_items,
hf_processor_mm_kwargs,
processed_data = super()._postprocess_hf_mm_data(
mm_data, hf_processor_mm_kwargs, processed_data
)
if (
"audio_feature_lengths" in processed_data
and "feature_attention_mask" in processed_data
):
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, _ = self._get_hf_mm_data(valid_mm_items)
audios = processor_data.get("audios", [])
audios = mm_data.get("audio", [])
if audios:
feature_extractor = self.info.get_feature_extractor(
**hf_processor_mm_kwargs
@@ -1312,12 +1310,14 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""Qwen3-Omni reimplements this function to handle `use_audio_in_video`."""
mm_item_counts = mm_items.get_all_counts()
prompt_ids = mm_res.prompt_ids
mm_kwargs = mm_res.kwargs
mm_prompt_updates = mm_res.prompt_updates
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
use_audio_in_video = False
+16 -22
View File
@@ -1327,27 +1327,25 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
return proc_impl is not None and proc_impl is not mixin_impl
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data)
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
# Separate video processing from image processing. Because the videos
# are processed into several image patches
video_input_ids_lst: list[list[int]] = []
if videos := mm_data.pop("videos", []):
if videos := hf_data.pop("videos", []):
video_grid_thw_lst = []
pixel_values_videos_lst = []
timestamps_per_video = []
@@ -1370,10 +1368,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# NOTE: a copy of is created to update do_sample_frames,
# otherwise mm_hash for the object will be incorrect.
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(
hf_processor_mm_kwargs, modality="video"
)
video_mm_kwargs = dict(**hf_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
video_size = dict(self.info.get_video_processor().size)
size_override = merged.get("size")
@@ -1485,13 +1481,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# fps/num_frames are video-only kwargs already consumed by the loop;
# exclude them so the text/image processor call below never gets a list.
non_video_mm_kwargs = {
k: v
for k, v in hf_processor_mm_kwargs.items()
if k not in ("fps", "num_frames")
k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames")
}
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**non_video_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_data,
non_video_mm_kwargs,
)
@@ -1516,9 +1510,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
processed_data["input_ids"] = [expanded_ids]
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
+3 -1
View File
@@ -202,7 +202,9 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
with timing_ctx.record("apply_hf_processor"):
_, passthrough_data = self._get_hf_mm_data(mm_items)
passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_processor_mm_kwargs
).passthrough_data
mm_processed_data = BatchFeature(
{
k: torch.as_tensor(v).unsqueeze(0)
@@ -219,18 +219,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
Subclasses add the strategy for locating placeholders in the prompt.
"""
def _get_hf_mm_data(
self,
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""Rename the parser's `audios` key to the `audio` argument HF audio
processors take.
"""
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
if self.info._is_audio_model() and "audios" in processor_data:
processor_data["audio"] = processor_data.pop("audios")
return processor_data, passthrough_data
def _get_modality_field_names(self, modality: str) -> set[str]:
"""Names of the fields the sub-processor for `modality` produces."""
# TODO: use else branch only once huggingface/transformers#44394 lands.
@@ -353,6 +341,9 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
return mm_fields
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _unpad_audios(
self,
hf_inputs: "BatchFeature",
@@ -597,21 +588,31 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
# work for Transformers. The vision path has logic tied to
# `mm_tokens_per_modality` in _apply_vision()
hf_processor_mm_kwargs = {
**hf_processor_mm_kwargs,
# vLLM needs the untruncated sequence to keep placeholder
# tokens aligned. Note that the text inputs are just dummy
# text, not the original prompt. The original prompt is
# already tokenized by the renderer.
"truncation": False,
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
# (the HF processor call disables `add_special_tokens`).
"add_special_tokens": False,
**hf_processor_mm_kwargs,
}
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
processor_data, _, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_processor_mm_kwargs
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
# The real prompt is fed to the processor below and the placeholder
# ranges are read back out of its output, so the dummy text meant for
# media-only callers is discarded here.
processor_data.pop("text", None)
has_mm_data = any(processor_data.values())
# Ask which modality owns each token of the expanded prompt, which
# is the only marking of the tokens an expansion adds around an item
if any(processor_data.values()):
if has_mm_data:
processor_data = {**processor_data, "return_mm_token_type_ids": True}
try:
@@ -621,7 +622,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
hf_processor_mm_kwargs,
)
except ValueError:
if any(processor_data.values()):
if has_mm_data:
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
@@ -792,31 +793,37 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
except (AttributeError, KeyError, TypeError):
return None
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> "BatchFeature":
mm_counts = mm_items.get_all_counts()
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
# Ask for the replacement each placeholder expands to, and record it as
# per-item fields: its token ids, and the tokens or patches behind them
if has_mm_data := any(mm_data.values()):
mm_data = {**mm_data, "return_text_replacement_offsets": True}
if has_mm_data := any(hf_data.values()):
hf_data = {**hf_data, "return_text_replacement_offsets": True}
try:
hf_inputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
self.info.get_hf_processor(**hf_kwargs),
dict(text=prompt_text, **hf_data),
hf_kwargs,
)
except ValueError:
if any(mm_data.values()):
if any(hf_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
@@ -825,7 +832,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)
self._unpad_audios(hf_inputs, hf_data, hf_kwargs)
# Drop the inputs the model would reject
hf_inputs.pop("mm_token_type_ids", None)
@@ -845,9 +852,10 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"so it can be fixed, and install `transformers<5.15.0` in the "
"meantime to locate placeholders in the expanded prompt instead."
)
hf_inputs.update(passthrough_data)
hf_inputs.pop("input_ids", None)
return hf_inputs
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, hf_inputs
)
tokenizer = self.info.get_tokenizer()
replacements = defaultdict[str, list[list[int]]](list)
@@ -865,7 +873,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
)
if modality == "image":
hf_inputs["num_image_patches"] = self._get_num_image_patches(
hf_inputs, mm_data, len(seqs)
hf_inputs, hf_data, len(seqs)
)
elif modality == "audio":
counts = []
@@ -874,10 +882,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
hf_inputs.update(passthrough_data)
hf_inputs.pop("input_ids")
return hf_inputs
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, hf_inputs
)
# From this version on, a processor reporting no offsets is an error rather than a
+21 -60
View File
@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
cached_encode,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.renderers import TokenizeParams
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
@@ -190,72 +191,32 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
audios = mm_data.get("audios", [])
assert isinstance(audios, list)
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
include_audio_num_chunks=True,
)
return mm_data, hf_processor_mm_kwargs
def _apply_hf_processor_main(
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
if processor_data:
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
processor_data,
hf_processor_mm_kwargs,
# The Ultravox processor accepts "audios" instead of "audio"
hf_data = hf_inputs.hf_data
if "audio" in hf_data:
hf_data["audios"] = hf_data.pop("audio")
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
normalized_kwargs = dict(hf_inputs.hf_kwargs)
# The remote processor already hardcodes truncation=False and forwards
# its kwargs to the audio processor, where another value would collide.
normalized_kwargs.pop("truncation", None)
return hf_inputs._replace(
hf_kwargs=dict(
normalized_kwargs,
sampling_rate=feature_extractor.sampling_rate,
include_audio_num_chunks=True,
)
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
if prompt_text is not None:
processor_data = dict(text=prompt_text, **processor_data)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
def patched_call(**kwargs) -> BatchFeature:
# The remote UltravoxProcessor does not support the
# `truncation` kwarg injected by `call_hf_processor`: it
# passes `truncation` to its audio processor by itself while
# also forwarding `**kwargs` there, causing a duplicate
# keyword argument error. Its hardcoded `truncation=False`
# already matches vLLM's intended behavior.
kwargs.pop("truncation", None)
return hf_processor(**kwargs)
processed_data = self.info.ctx.call_hf_processor(
patched_call,
processor_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
else:
processed_data = BatchFeature(dict(passthrough_data))
return self._postprocess_hf_mm_data(
processor_data,
hf_processor_mm_kwargs,
processed_data,
)
def _postprocess_hf_mm_data(
+2 -2
View File
@@ -75,7 +75,7 @@ from vllm.multimodal.parse import (
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
from vllm.multimodal.processing.context import TimingContext
from vllm.multimodal.processing.inputs import ProcessorInputs
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
from vllm.multimodal.processing.processor import MultiModalProcessingResult
from vllm.transformers_utils.processors.deepseek_ocr import (
BASE_SIZE,
CROP_MODE,
@@ -225,7 +225,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
# The processor logic differs for single-image (crop) vs multi-image
# (no crop) requests. The processing cache assumes per-item output is
# invariant of how many images are passed per prompt, so we only cache
+18 -39
View File
@@ -16,7 +16,7 @@ from mistral_common.protocol.instruct.messages import UserMessage
from mistral_common.protocol.instruct.request import ChatCompletionRequest
from mistral_common.protocol.transcription.request import TranscriptionRequest
from mistral_common.tokens.tokenizers.audio import Audio
from transformers import BatchFeature, WhisperConfig
from transformers import WhisperConfig
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
@@ -36,7 +36,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItems,
MultiModalKwargsOptionalItems,
NestedTensors,
)
from vllm.multimodal.parse import (
@@ -48,7 +47,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
MultiModalPromptUpdates,
HFMultiModalInputs,
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
ProcessorInputs,
PromptReplacement,
@@ -61,7 +61,6 @@ from vllm.transformers_utils.processors.voxtral import (
MistralCommonFeatureExtractor,
MistralCommonVoxtralProcessor,
)
from vllm.utils.collection_utils import is_list_of
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription
from .utils import init_vllm_registered_model, maybe_prefix
@@ -209,21 +208,19 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsOptionalItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_item_counts = mm_items.get_all_counts()
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
mm_placeholders = self._find_mm_placeholders(
prompt_ids,
mm_prompt_updates,
mm_res.prompt_ids,
mm_res.prompt_updates,
)
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
return prompt_ids, mm_placeholders
return mm_res.prompt_ids, mm_placeholders
def _get_mm_fields_config(
self,
@@ -241,38 +238,20 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
# skip validation here
pass
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
# MistralCommonVoxtralProcessor accepts "audio"
mm_data["audio"] = audios
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# Avoid padding issue
hf_processor_mm_kwargs = dict(**hf_processor_mm_kwargs, return_tensors=None)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
# Missing batch dimension
input_ids = processed_data.get("input_ids")
if is_list_of(input_ids, int):
processed_data["input_ids"] = [input_ids]
return processed_data
return hf_inputs._replace(
hf_kwargs=dict(hf_inputs.hf_kwargs, return_tensors=None)
)
def _get_prompt_updates(
self,
@@ -28,11 +28,10 @@ from vllm.model_executor.models.voxtral import (
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
from vllm.multimodal.inputs import MultiModalKwargsOptionalItems
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
MultiModalPromptUpdates,
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
)
from vllm.sequence import IntermediateTensors
@@ -60,10 +59,10 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsOptionalItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_kwargs = mm_res.kwargs
# there are no placeholder audio tokens for streaming
# so we need to build the place placeholder positions manually
@@ -86,7 +85,7 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
* [0], # only used for length computation, so we can take dummy inputs
is_embed=None,
)
return prompt_ids, {"audio": [features_info]}
return mm_res.prompt_ids, {"audio": [features_info]}
class TimeEmbedding(torch.nn.Module):
+15 -27
View File
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.renderers import TokenizeParams
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.jsontree import json_map_leaves
@@ -743,6 +744,9 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def create_encoder_prompt(
self,
prompt: list[int],
@@ -754,37 +758,21 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
# for encoder profiling.
return [0]
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
return hf_inputs._replace(
hf_kwargs=dict(
hf_inputs.hf_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,