[MM] Simplify _apply_hf_processor_main (#53275)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-22 01:49:37 -07:00
committed by GitHub
parent 7ca49fbe4b
commit 0b19ebcacd
90 changed files with 1848 additions and 1706 deletions
+27 -20
View File
@@ -377,35 +377,41 @@ class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):
class AudioFlamingo3MultiModalProcessor(
BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo]
):
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any],
) -> BatchFeature:
processor_mm_data = dict(mm_data)
audios = processor_mm_data.pop("audios", None)
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", None)
if audios is not None:
processor_mm_data["audio"] = audios
mm_data["audio"] = audios
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=processor_mm_data,
mm_kwargs=mm_kwargs,
)
return mm_data, hf_processor_mm_kwargs
if "input_features_mask" in outputs:
outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "input_features_mask" in processed_data:
processed_data["feature_attention_mask"] = processed_data.pop(
"input_features_mask"
)
audio_data = processor_mm_data.get("audio")
audio_data = mm_data.get("audio")
if audio_data is None:
return outputs
return processed_data
audio_list = audio_data if isinstance(audio_data, list) else [audio_data]
if len(audio_list) == 0:
return outputs
return processed_data
processor = self.info.get_hf_processor(**mm_kwargs)
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
feature_extractor = processor.feature_extractor
sampling_rate = feature_extractor.sampling_rate
chunk_length = feature_extractor.chunk_length
@@ -423,8 +429,9 @@ class AudioFlamingo3MultiModalProcessor(
n_win = max_windows
chunk_counts.append(n_win)
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
return outputs
processed_data["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
return processed_data
def _get_mm_fields_config(
self,
-18
View File
@@ -467,24 +467,6 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
# HF processor always adds placeholders even when there's no image
tokenizer = self.info.get_tokenizer()
prompt_ids = tokenizer.encode(prompt)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
return super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+6 -24
View File
@@ -135,34 +135,16 @@ class ChameleonDummyInputsBuilder(BaseDummyInputsBuilder[ChameleonProcessingInfo
class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
# ChameleonProcessor requires text corresponding to the images
return self.dummy_inputs.get_dummy_text(mm_counts)
return super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _apply_hf_processor_tokens_only(
self,
prompt_tokens: list[int],
) -> list[int]:
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
# HF processor adds sep token for chat mode
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
sep_token_id = tokenizer.get_vocab()[tokenizer.sep_token] # type: ignore
sep_token_id = vocab[tokenizer.sep_token] # type: ignore
return prompt_tokens + [sep_token_id]
return [*prompt, sep_token_id]
def _get_mm_fields_config(
self,
-9
View File
@@ -16,7 +16,6 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
@@ -492,14 +491,6 @@ class CheersDummyInputsBuilder(BaseDummyInputsBuilder[CheersProcessingInfo]):
class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):
"""Multimodal processor for Cheers model."""
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
+13 -14
View File
@@ -222,24 +222,23 @@ class Cohere2VisionDummyInputsBuilder(
class Cohere2VisionMultiModalProcessor(
BaseMultiModalProcessor[Cohere2VisionProcessingInfo]
):
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt,
mm_data,
mm_kwargs,
)
if not mm_data:
return processed_data
# Ensure num_patches is available for proper tensor splitting
if (
"num_patches" not in processed_outputs
"num_patches" not in processed_data
and (images := mm_data.get("images")) is not None
):
hf_processor = self.info.get_hf_processor(**mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
# Fallback calculation if HF processor didn't provide num_patches
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
@@ -250,13 +249,13 @@ class Cohere2VisionMultiModalProcessor(
image_width=parsed_images.get_image_size(i).width,
image_height=parsed_images.get_image_size(i).height,
processor=hf_processor,
mm_kwargs=mm_kwargs,
mm_kwargs=hf_processor_mm_kwargs,
)
for i in range(len(parsed_images))
]
processed_outputs["num_patches"] = torch.tensor(num_patches)
processed_data["num_patches"] = torch.tensor(num_patches)
return processed_outputs
return processed_data
def _get_mm_fields_config(
self,
+46 -19
View File
@@ -8,7 +8,7 @@ from typing import Any, ClassVar
import torch
import torch.nn.functional as F
from torch import nn
from transformers import PretrainedConfig
from transformers import BatchFeature, PretrainedConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
@@ -45,9 +45,12 @@ from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseProcessingInfo,
EncDecMultiModalProcessor,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
TimingContext,
)
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
from vllm.renderers import TokenizeParams
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.transformers_utils.processors.cohere_asr import (
@@ -1941,27 +1944,51 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
) -> list[int]:
return [0]
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
):
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_data = dict(audio=dict(mm_data).pop("audios"))
mm_kwargs = dict(
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
return processed_outputs
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
# Dithering injects noise into the extracted features, so the
# feature extractor is not a pure function of its input. Since the
# processing cache assumes that processor outputs are invariant
# across calls, bypass the cache when dithering is active.
preproc = self.info.get_hf_config().preprocessor
if preproc.get("dither", 1e-05) > 0:
return self._apply_hf_processor(inputs, timing_ctx)
return super()._cached_apply_hf_processor(inputs, timing_ctx)
def _get_mm_fields_config(
self,
+17 -6
View File
@@ -154,16 +154,25 @@ class ColModernVBertDummyInputsBuilder(
class ColModernVBertMultiModalProcessor(
BaseMultiModalProcessor[ColModernVBertProcessingInfo],
):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
text_encoding = tokenizer(
prompt,
prompt_text,
return_tensors="pt",
)
result = BatchFeature(data=dict(text_encoding))
@@ -183,7 +192,9 @@ class ColModernVBertMultiModalProcessor(
)
result.update(image_outputs)
return result
processed_data = result
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
+24 -16
View File
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import MultiModalDataItems
from .interfaces import SupportsLateInteraction
from .interfaces_base import default_pooling_type
@@ -59,24 +60,31 @@ class ColPaliProcessingInfo(PaliGemmaProcessingInfo):
class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
"""Multimodal processor for ColPali."""
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
# The ColPali tokenizer_config.json ships with a small default
# max_length (50) that truncates the 1024 image tokens inserted
# by PaliGemmaProcessor, causing a token-count mismatch.
# vLLM enforces its own max_model_len, so we disable HF
# truncation to keep all image + text tokens intact.
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
dict(**mm_kwargs, truncation=False),
)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
# The ColPali tokenizer_config.json ships with a small default
# max_length (50) that truncates the 1024 image tokens inserted
# by PaliGemmaProcessor, causing a token-count mismatch.
# vLLM enforces its own max_model_len, so we disable HF
# truncation to keep all image + text tokens intact.
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
dict(**hf_processor_mm_kwargs, truncation=False),
)
processed_data.update(passthrough_data)
return processed_data
@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
+18 -11
View File
@@ -287,27 +287,34 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing
class DeepseekOCRMultiModalProcessor(
BaseMultiModalProcessor[DeepseekOCRProcessingInfo]
):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
if mm_data:
processed_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(prompt=prompt, **mm_data),
mm_kwargs,
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(prompt=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
else:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
processed_outputs = tokenizer(
prompt, add_special_tokens=True, return_tensors="pt"
processed_data = tokenizer(
prompt_text, add_special_tokens=True, return_tensors="pt"
)
return processed_outputs
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
+19 -11
View File
@@ -158,27 +158,35 @@ class DeepseekOCR2DummyInputsBuilder(
class DeepseekOCR2MultiModalProcessor(
BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo]
):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
if mm_data:
processed_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(prompt=prompt, **mm_data),
mm_kwargs,
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(prompt=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
else:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
processed_outputs = tokenizer(
prompt, add_special_tokens=True, return_tensors="pt"
processed_data = tokenizer(
prompt_text, add_special_tokens=True, return_tensors="pt"
)
return processed_outputs
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
+11 -17
View File
@@ -43,7 +43,6 @@ from vllm.multimodal.processing.processor import (
)
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.hf import HfTokenizer
from vllm.transformers_utils.configs.deepseek_vl2 import (
DeepseekVLV2Config,
MlpProjectorConfig,
@@ -236,28 +235,23 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
class DeepseekVL2MultiModalProcessor(
BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]
):
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
return tokenizer(prompt, add_special_tokens=True, return_tensors="pt")
return processed_data
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
processed_data["num_patches"] = (
processed_data["images_spatial_crop"].prod(-1) + 1
)
processed_outputs["num_patches"] = (
processed_outputs["images_spatial_crop"].prod(-1) + 1
)
return processed_outputs
return processed_data
def _get_mm_fields_config(
self,
@@ -315,7 +309,7 @@ class DeepseekVL2MultiModalProcessor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
) -> MultiModalProcessingInfo:
# The processor logic is different for len(images) <= 2 vs > 2
# Since the processing cache assumes that the processor output is
# invariant of how many images are passed per prompt, we only
+21 -18
View File
@@ -1110,21 +1110,20 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
pixel_values = pixel_values.to(hf_config.dtype)
return pixel_values
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# when the prompt is not empty but the multimodal data is empty,
# directly invoke the tokenizer.
if "images" not in mm_data and "videos" not in mm_data and prompt != "":
tokenizer = self.info.get_tokenizer()
prompt_ids = tokenizer.encode(prompt)
tokenizer_output = BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
return tokenizer_output
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if "images" not in mm_data and "videos" not in mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
if "images" not in mm_data:
mm_data["images"] = []
@@ -1132,7 +1131,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
mm_data["videos"] = []
# Check if HF processor supports video metadata
hf_processor = self.info.get_hf_processor(**mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
supports_video_metadata = getattr(
hf_processor, "supports_video_metadata", False
)
@@ -1149,8 +1148,10 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
processor_output = self.info.ctx.call_hf_processor(
hf_processor,
dict(text=[prompt], images=mm_data["images"], videos=mm_data["videos"]),
mm_kwargs,
dict(
text=[prompt_text], images=mm_data["images"], videos=mm_data["videos"]
),
hf_processor_mm_kwargs,
)
# Divide the processor_output into two modalities: image and video.
@@ -1158,7 +1159,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
pixel_values = processor_output["images"]
if pixel_values is not None:
processor_output["images"] = self._pixel_values_norm(
pixel_values, mm_kwargs
pixel_values, hf_processor_mm_kwargs
)
for key in list(processor_output.keys()):
if processor_output[key] is None:
@@ -1184,7 +1185,9 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
]
del processor_output["images"]
return processor_output
processed_data = processor_output
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates(
self,
+27 -18
View File
@@ -228,27 +228,36 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing
class FireRedASR2MultiModalProcessor(
BaseMultiModalProcessor[FireRedASR2ProcessingInfo]
):
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_data = dict(audio=mm_data.pop("audios"))
mm_kwargs = dict(
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
return processed_outputs
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _get_mm_fields_config(
self,
+27 -18
View File
@@ -451,27 +451,36 @@ class FireRedLIDMultiModalProcessor(
# Dummy encoder prompt for profiling (encoder only processes audio).
return [0]
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_data = dict(audio=mm_data.pop("audios"))
mm_kwargs = dict(
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
return processed_outputs
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _get_mm_fields_config(
self,
+44 -18
View File
@@ -44,9 +44,12 @@ from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
TimingContext,
)
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -747,27 +750,50 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_data = dict(audio=mm_data.pop("audios"))
mm_kwargs = dict(
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
return processed_outputs
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
# Dithering injects noise into the extracted features, so the
# feature extractor is not a pure function of its input. Since the
# processing cache assumes that processor outputs are invariant
# across calls, bypass the cache when dithering is active.
if self.info.get_feature_extractor().dither > 0:
return self._apply_hf_processor(inputs, timing_ctx)
return super()._cached_apply_hf_processor(inputs, timing_ctx)
def _get_mm_fields_config(
self,
+21 -8
View File
@@ -612,20 +612,31 @@ class FunAudioChatDummyInputsBuilder(
class FunAudioChatMultiModalProcessor(
BaseMultiModalProcessor[FunAudioChatProcessingInfo]
):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
tokenizer = self.info.get_tokenizer()
input_ids = torch.tensor([tokenizer.encode(prompt)])
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
audios = mm_data.get("audios", [])
if not audios:
return BatchFeature({"input_ids": input_ids})
processed_data = BatchFeature({"input_ids": input_ids})
processed_data.update(passthrough_data)
return processed_data
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
sr = int(feature_extractor.sampling_rate)
min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400)
@@ -676,7 +687,9 @@ class FunAudioChatMultiModalProcessor(
"feature_exist_mask": torch.ones((len(wavs),), dtype=torch.bool),
}
return BatchFeature({"input_ids": input_ids, **mm_inputs})
processed_data = BatchFeature({"input_ids": input_ids, **mm_inputs})
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
+12 -13
View File
@@ -261,39 +261,38 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt,
mm_data,
mm_kwargs,
)
if not mm_data:
return processed_data
# HF processor pops the `num_crops` kwarg, which is needed by vLLM
if (images := mm_data.get("images")) is not None:
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [
parsed_images.get_image_size(i) for i in range(len(parsed_images))
]
hf_processor = self.info.get_hf_processor(**mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
num_crops = [
self.info.get_num_crops(
image_width=size.width,
image_height=size.height,
processor=hf_processor,
mm_kwargs=mm_kwargs,
mm_kwargs=hf_processor_mm_kwargs,
)
for size in image_sizes
]
processed_outputs["num_patches"] = torch.tensor(num_crops) + 1
processed_data["num_patches"] = torch.tensor(num_crops) + 1
return processed_outputs
return processed_data
def _get_mm_fields_config(
self,
+57 -27
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import inspect
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal
@@ -257,40 +258,69 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]):
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# HF Transformers audio processor no longer accepts `audios` key.
# We pop `audios` and replace it with `audio` key to suppress
# the warning.
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
if "audios" in mm_data:
mm_data["audio"] = mm_data.pop("audios")
processed_outputs = super()._call_hf_processor(
prompt,
mm_data,
mm_kwargs,
)
if "input_features" in processed_outputs:
# Padding enables audio_tower to run in batched mode
processed_outputs["input_features_padded"] = processed_outputs[
"input_features"
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "input_features" in processed_data:
# The feature extractor pads every clip to the longest one in
# the batch, so each clip's frame count depends on the other
# clips it was processed with. Trim each clip to the number of
# frames it would have if processed alone so that each item's
# output (and thus its cache entry) is independent of the other
# items in the batch. `batch_audio_features` re-pads the
# per-item features so audio_tower can still run batched.
num_frames = [
self._get_num_audio_frames_alone(len(audio))
for audio in mm_data["audio"]
]
processed_data["input_features_padded"] = [
features[:n]
for features, n in zip(processed_data.pop("input_features"), num_frames)
]
processed_data["input_features_mask"] = [
mask[:n]
for mask, n in zip(processed_data["input_features_mask"], num_frames)
]
# Unpad features here since we need the output of each item to be
# independent of other items for the cache to work correctly
unpadded_features = [
f[mask]
for f, mask in zip(
processed_outputs["input_features"],
processed_outputs["input_features_mask"],
)
]
processed_outputs["input_features"] = unpadded_features
return processed_outputs
return processed_data
def _get_num_audio_frames_alone(self, num_samples: int) -> int:
"""
Get the number of mel frames that the feature extractor produces
for a single audio clip of the given length, independent of batch
padding.
"""
feature_extractor = self.info.get_feature_extractor()
call_params = inspect.signature(type(feature_extractor).__call__).parameters
max_length = call_params["max_length"].default
pad_to_multiple_of = call_params["pad_to_multiple_of"].default
num_samples = min(num_samples, max_length)
remainder = num_samples % pad_to_multiple_of
if remainder:
num_samples += pad_to_multiple_of - remainder
frame_size = feature_extractor.frame_length + 1
num_frames = (num_samples - frame_size) // feature_extractor.hop_length + 1
return max(num_frames, 0)
def _get_mm_fields_config(
self,
+38 -28
View File
@@ -560,13 +560,22 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
if val is not None and val not in _SUPPORTED_SOFT_TOKENS:
@@ -611,15 +620,15 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
timestamps = [idx / fps for idx in frame_indices]
# Process frames as images with max_soft_tokens=70
video_mm_kwargs = dict(mm_kwargs)
video_mm_kwargs = dict(hf_processor_mm_kwargs)
video_mm_kwargs["max_soft_tokens"] = _VIDEO_MAX_SOFT_TOKENS
dummy_prompt = ("\t" + processor.image_token) * len(frames)
frame_outputs = super()._call_hf_processor(
prompt=dummy_prompt,
mm_data={"images": frames},
mm_kwargs=video_mm_kwargs,
frame_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**video_mm_kwargs),
dict(text=dummy_prompt, **{"images": frames}),
video_mm_kwargs,
)
# Remap HF key name
@@ -660,7 +669,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
# split-replace bug where replacement text (which itself
# contains <|video|> tokens) collides with later splits.
vt = processor.video_token
parts = prompt.split(vt, len(video_replacements))
parts = prompt_text.split(vt, len(video_replacements))
# NOTE: len(parts) <= len(video_replacements) + 1
parts_with_repl: list[str] = []
@@ -668,7 +677,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
parts_with_repl.extend([part, repl])
parts_with_repl.extend(parts[len(video_replacements) :])
prompt = "".join(parts_with_repl)
prompt_text = "".join(parts_with_repl)
video_outputs = {
"pixel_values_videos": torch.cat(all_video_pixel_values, dim=0),
@@ -712,45 +721,46 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
# NOTE: This requires a corresponding type annotation on the
# HF side (Gemma4ProcessorKwargs.images_kwargs) so that
# _merge_kwargs routes max_soft_tokens into images_kwargs.
patched_mm_kwargs = dict(mm_kwargs)
patched_mm_kwargs = dict(hf_processor_mm_kwargs)
if val is not None and is_top_level_max_soft_tokens:
patched_mm_kwargs["max_soft_tokens"] = val
processed_outputs = super()._call_hf_processor(
prompt,
mm_data,
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**patched_mm_kwargs),
dict(text=prompt_text, **mm_data),
patched_mm_kwargs,
)
# HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'.
# Remap here to keep a single translation point.
if "image_position_ids" in processed_outputs:
processed_outputs["pixel_position_ids"] = processed_outputs.pop(
if "image_position_ids" in processed_data:
processed_data["pixel_position_ids"] = processed_data.pop(
"image_position_ids"
)
if "input_features" in processed_outputs:
if "input_features" in processed_data:
# Unpad per-item so each item's cache entry is
# self-contained. The batched() field config in
# _get_mm_fields_config stacks equal-length items and leaves
# the rest as a list; _process_audio_input re-pads that list
# to the batch's max length.
masks = processed_outputs["input_features_mask"]
masks = processed_data["input_features_mask"]
unpadded_features = [
f[mask]
for f, mask in zip(
processed_outputs["input_features"],
processed_data["input_features"],
masks,
)
]
unpadded_masks = [mask[mask] for mask in masks]
processed_outputs["input_features"] = unpadded_features
processed_outputs["input_features_padded"] = unpadded_features
processed_outputs["input_features_mask"] = unpadded_masks
processed_data["input_features"] = unpadded_features
processed_data["input_features_padded"] = unpadded_features
processed_data["input_features_mask"] = unpadded_masks
# Merge video outputs into the final result
combined_outputs = dict(processed_outputs, **video_outputs)
return BatchFeature(combined_outputs)
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
@@ -815,7 +825,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
image_size = images.get_image_size(item_idx)
# Resolve the effective max_soft_tokens by merging
# per-prompt kwargs with the config-level defaults,
# consistent with how _call_hf_processor resolves it.
# consistent with how _apply_hf_processor_main resolves it.
# Without this merge, a missing per-prompt override
# would fall back to vision_cfg.default_output_length
# instead of the config's mm_processor_kwargs default.
+42 -32
View File
@@ -1576,32 +1576,39 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
return prepared_data, prepared_kwargs
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
if not mm_data:
tokenizer = self.info.get_tokenizer()
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
processor = self.info.get_hf_processor(**mm_kwargs)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data)
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
use_direct_path = (
not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
)
if use_direct_path:
prepared_data, prepared_kwargs = self._get_direct_path_inputs(
mm_data, mm_kwargs
mm_data, hf_processor_mm_kwargs
)
return super()._call_hf_processor(
prompt=prompt,
mm_data=prepared_data,
mm_kwargs=prepared_kwargs,
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**prepared_kwargs),
dict(text=prompt_text, **prepared_data),
prepared_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
if (
"videos" in mm_data
@@ -1616,7 +1623,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
video_array, metadata = item
# don't update mm_kwargs inplace
video_mm_kwargs = dict(**mm_kwargs)
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
video_mm_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", True
)
@@ -1625,10 +1632,13 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
video_mm_data["videos"] = [[video_array]]
video_mm_data["video_metadata"] = [[_to_video_metadata(metadata)]]
video_outputs = super()._call_hf_processor(
prompt="<|begin_of_video|><|video|><|end_of_video|>",
mm_data=video_mm_data,
mm_kwargs=video_mm_kwargs,
video_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**video_mm_kwargs),
dict(
text="<|begin_of_video|><|video|><|end_of_video|>",
**video_mm_data,
),
video_mm_kwargs,
)
input_ids = video_outputs.pop("input_ids")
if swap_video_frame_tokens:
@@ -1636,7 +1646,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
processor.video_token_id
)
video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
prompt = prompt.replace(
prompt_text = prompt_text.replace(
"<|begin_of_video|><|video|><|end_of_video|>",
video_placeholder,
1,
@@ -1652,20 +1662,20 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
video_outputs = dict()
swap_video_frame_tokens = False
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
if swap_video_frame_tokens:
input_ids = processed_outputs["input_ids"]
input_ids = processed_data["input_ids"]
input_ids[input_ids == processor.video_token_id] = processor.image_token_id
processed_outputs["input_ids"] = input_ids
combined_outputs = dict(
processed_outputs,
**video_outputs,
)
return BatchFeature(combined_outputs)
processed_data["input_ids"] = input_ids
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
+37 -39
View File
@@ -750,46 +750,40 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
chunk_counts.append(min(n_chunks, max_windows))
return chunk_counts
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
prompt: str,
mm_data: dict[str, object],
mm_kwargs: Mapping[str, Any],
) -> BatchFeature:
# Normalize input: handle deprecated key and list conversion.
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
if "audios" in mm_data:
mm_data["audio"] = mm_data.pop("audios")
audio = mm_data.get("audio", [])
audio_list = [audio] if audio and not isinstance(audio, list) else audio
# Early return for text-only.
if not audio_list:
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
# Handle sampling_rate
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_kwargs = dict(
**mm_kwargs,
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
# Call parent method
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
return mm_data, hf_processor_mm_kwargs
# Postprocess: rename mask and add chunk counts
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
# Handle different key names from different transformers versions
if "input_features_mask" in outputs:
outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
elif "input_features_mask" not in outputs and "input_features" in outputs:
if "input_features_mask" in processed_data:
processed_data["feature_attention_mask"] = processed_data.pop(
"input_features_mask"
)
elif "input_features" in processed_data:
# If no mask is provided, create one from input_features
input_features = outputs["input_features"]
input_features = processed_data["input_features"]
if isinstance(input_features, torch.Tensor):
# Create a mask of all ones matching the sequence length
mask = torch.ones(
@@ -797,18 +791,22 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
input_features.shape[-1],
dtype=torch.long,
)
outputs["feature_attention_mask"] = mask
processed_data["feature_attention_mask"] = mask
# Get processor for chunk counts calculation
processor = self.info.get_hf_processor(**mm_kwargs)
audio = mm_data.get("audio", [])
audio_list = [audio] if audio and not isinstance(audio, list) else audio
if audio_list:
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
# Override chunk counts calculation with GLM-ASR specific logic
chunk_counts = self._calculate_chunk_counts(
audio_list, processor.feature_extractor, processor
)
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
# Override chunk counts calculation with GLM-ASR specific logic
chunk_counts = self._calculate_chunk_counts(
audio_list, processor.feature_extractor, processor
)
processed_data["chunk_counts"] = torch.tensor(
chunk_counts, dtype=torch.long
)
return outputs
return processed_data
def _get_mm_fields_config(
self,
+16 -12
View File
@@ -180,12 +180,14 @@ class GraniteSpeechMultiModalProcessor(
)
]
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -193,21 +195,23 @@ class GraniteSpeechMultiModalProcessor(
# GraniteSpeechFeatureExtractor accepts "audio"
mm_data["audio"] = audios
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "audio" in mm_data:
# Calculate the number of audio tokens per entry in the batch;
# This is used to split the batch back out after padding.
audio_token_index = self.info.get_hf_config().audio_token_index
processed_outputs["audio_embed_sizes"] = (
processed_outputs["input_ids"] == audio_token_index
processed_data["audio_embed_sizes"] = (
processed_data["input_ids"] == audio_token_index
).sum(-1)
return processed_outputs
return processed_data
class GraniteSpeechDummyInputsBuilder(
+1 -1
View File
@@ -135,7 +135,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
) -> MultiModalProcessingInfo:
# The processor logic is different for len(images) <= 1 vs > 1
# Since the processing cache assumes that the processor output is
# invariant of how many images are passed per prompt, we only
@@ -731,28 +731,6 @@ class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo
class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
hf_processor = self.info.get_hf_processor(**mm_kwargs)
# HunYuanVLProcessor requires image placeholders wrapped with start/end tokens.
if mm_data.get("images") is not None and prompt:
img_tok = hf_processor.image_token
wrapped = (
f"{hf_processor.image_start_token}{img_tok}"
f"{hf_processor.image_end_token}"
)
if img_tok in prompt and wrapped not in prompt:
prompt = prompt.replace(img_tok, wrapped)
return self.info.ctx.call_hf_processor(
hf_processor,
dict(text=prompt, **mm_data),
mm_kwargs,
)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
@@ -190,77 +190,71 @@ class HCXVisionDummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionProcessingInfo
class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingInfo]):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
for video_idx, video_arr in enumerate(mm_data.get("videos", [])):
if video_arr.dtype != np.uint8:
mm_data["videos"][video_idx] = video_arr.astype(np.uint8)
processed_outputs = self.info.ctx.call_hf_processor(
hf_processor=self.info.get_hf_processor(**mm_kwargs),
images = mm_data.get("images")
videos = mm_data.get("videos")
# batchify input as a single item
processed_data = self.info.ctx.call_hf_processor(
hf_processor=self.info.get_hf_processor(**hf_processor_mm_kwargs),
data=dict(
text=prompt,
images=None,
videos=None,
images=None if images is None else [images],
videos=None if videos is None else [videos],
),
) # text-only
)
if len(mm_data) > 0:
images = mm_data.get("images")
videos = mm_data.get("videos")
for k, v in processed_data.items():
if isinstance(v, list) and len(v) > 0:
assert len(v) == 1
processed_data[k] = v[0]
# batchify input as a single item
_processed_outputs = self.info.ctx.call_hf_processor(
hf_processor=self.info.get_hf_processor(**mm_kwargs),
data=dict(
text=None,
images=None if images is None else [images],
videos=None if videos is None else [videos],
),
) # mm-only
if images:
processed_data["image_sizes_images"] = torch.tensor(
processed_data["image_sizes_images"]
)
processed_data["vision_query_lengths_images"] = torch.tensor(
processed_data["vision_query_lengths_images"]
)
for k, v in _processed_outputs.items():
if isinstance(v, list) and len(v) > 0:
assert len(v) == 1
_processed_outputs[k] = v[0]
if images:
_processed_outputs["image_sizes_images"] = torch.tensor(
_processed_outputs["image_sizes_images"]
)
_processed_outputs["vision_query_lengths_images"] = torch.tensor(
_processed_outputs["vision_query_lengths_images"]
)
if videos:
_idx_per_video = [
0,
*accumulate(
get_num_combined_frames(len(video)) for video in videos
),
if videos:
_idx_per_video = [
0,
*accumulate(get_num_combined_frames(len(video)) for video in videos),
]
processed_data["pixel_values_videos"] = [
processed_data["pixel_values_videos"][
_idx_per_video[i] : _idx_per_video[i + 1]
]
_processed_outputs["pixel_values_videos"] = [
_processed_outputs["pixel_values_videos"][
for i in range(len(videos))
]
processed_data["vision_query_lengths_videos"] = [
torch.tensor(
processed_data["vision_query_lengths_videos"][
_idx_per_video[i] : _idx_per_video[i + 1]
]
for i in range(len(videos))
]
_processed_outputs["vision_query_lengths_videos"] = [
torch.tensor(
_processed_outputs["vision_query_lengths_videos"][
_idx_per_video[i] : _idx_per_video[i + 1]
]
)
for i in range(len(videos))
]
)
for i in range(len(videos))
]
processed_outputs.update(_processed_outputs)
processed_data.update(passthrough_data)
return processed_outputs
return processed_data
def _get_prompt_updates(
self,
@@ -261,34 +261,44 @@ class HCXVisionV2MultiModalProcessor(
):
"""Multimodal processor for HyperCLOVAX V2 (32B Think model)."""
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
images = mm_data.get("images")
videos = mm_data.get("videos")
# Get the HF processor
hf_processor = self.info.get_hf_processor(**mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
# Build data dict for HF processor (images/videos only)
# NOTE: We pass the prompt as-is without token normalization.
# Token expansion is handled by vLLM via _get_prompt_updates.
data: dict[str, object] = dict(
text=prompt,
text=prompt_text,
images=images,
videos=videos,
)
processed_outputs = self.info.ctx.call_hf_processor(
processed_data = self.info.ctx.call_hf_processor(
hf_processor=hf_processor,
data=data,
kwargs=mm_kwargs,
kwargs=hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_outputs
return processed_data
def _get_prompt_updates(
self,
+26 -19
View File
@@ -239,51 +239,58 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Text-only input not supported in composite processor
if not (images := mm_data.get("images", [])):
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
image_processor = self.info.get_hf_processor().image_processor
if getattr(image_processor, "backend", "pil") == "pil":
mm_kwargs = {"input_data_format": "channels_last", **mm_kwargs}
hf_processor_mm_kwargs = {
"input_data_format": "channels_last",
**hf_processor_mm_kwargs,
}
processed_outputs = super()._call_hf_processor(
prompt,
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
mm_kwargs,
hf_processor_mm_kwargs,
)
images = mm_data.get("images", [])
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [
parsed_images.get_image_size(i) for i in range(len(parsed_images))
]
hf_processor = self.info.get_hf_processor(**mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
num_patches = [
self.info.get_num_patches(
image_width=size.width,
image_height=size.height,
processor=hf_processor,
mm_kwargs=mm_kwargs,
mm_kwargs=hf_processor_mm_kwargs,
)
for size in image_sizes
]
processed_outputs["num_patches"] = torch.tensor(num_patches)
processed_data["num_patches"] = torch.tensor(num_patches)
# Remove the extra batch dimension
processed_outputs["pixel_values"].squeeze_(0)
processed_outputs["pixel_attention_mask"].squeeze_(0)
processed_data["pixel_values"].squeeze_(0)
processed_data["pixel_attention_mask"].squeeze_(0)
return processed_outputs
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
+51 -25
View File
@@ -332,19 +332,28 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]):
"""Basic image-only MultiModalProcessor for InternS1-style models."""
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data)
videos = mm_data.pop("videos", [])
images = mm_data.pop("images", [])
assert isinstance(videos, list)
assert isinstance(images, list)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = hf_processor.tokenizer
video_token_id = tokenizer.encode(
hf_processor.video_token, add_special_tokens=False
@@ -352,23 +361,29 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
assert len(video_token_id) == 1
video_token_id = video_token_id[0]
prompt = re.sub(hf_processor.image_token, "<image_placeholder>", prompt)
prompt = re.sub(hf_processor.video_token, "<video_placeholder>", prompt)
prompt_text = re.sub(
hf_processor.image_token, "<image_placeholder>", prompt_text
)
prompt_text = re.sub(
hf_processor.video_token, "<video_placeholder>", prompt_text
)
image_outputs = {}
if images:
image_pixel_values = []
for image in images:
processed_outputs = super()._call_hf_processor(
prompt=hf_processor.image_token,
mm_data={"images": image},
mm_kwargs=mm_kwargs,
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=hf_processor.image_token, **{"images": image}),
hf_processor_mm_kwargs,
)
image_pixel_values.append(processed_outputs.pop("pixel_values"))
image_pixel_values.append(processed_data.pop("pixel_values"))
input_ids = processed_outputs.pop("input_ids")
input_ids = processed_data.pop("input_ids")
image_placeholder = tokenizer.batch_decode(input_ids)[0]
prompt = prompt.replace("<image_placeholder>", image_placeholder, 1)
prompt_text = prompt_text.replace(
"<image_placeholder>", image_placeholder, 1
)
num_patches = [len(item) for item in image_pixel_values]
image_outputs = {
@@ -381,18 +396,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
if videos:
video_pixel_values = []
for video in videos:
processed_outputs = super()._call_hf_processor(
prompt=hf_processor.video_token,
mm_data={"videos": video},
mm_kwargs=mm_kwargs,
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=hf_processor.video_token, **{"videos": video}),
hf_processor_mm_kwargs,
)
video_pixel_values.append(processed_outputs.pop("pixel_values"))
video_pixel_values.append(processed_data.pop("pixel_values"))
input_ids = processed_outputs.pop("input_ids")
input_ids = processed_data.pop("input_ids")
input_ids[input_ids == hf_processor.image_token_id] = video_token_id
video_placeholder = tokenizer.batch_decode(input_ids)[0]
prompt = prompt.replace("<video_placeholder>", video_placeholder, 1)
prompt_text = prompt_text.replace(
"<video_placeholder>", video_placeholder, 1
)
num_frames = [len(item) for item in video_pixel_values]
video_outputs = {
@@ -401,11 +418,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
"video_token_id": torch.tensor(video_token_id),
}
prompt = re.sub("<image_placeholder>", hf_processor.image_token, prompt)
prompt = re.sub("<video_placeholder>", hf_processor.video_token, prompt)
text_outputs = tokenizer(prompt, return_tensors="pt")
prompt_text = re.sub(
"<image_placeholder>", hf_processor.image_token, prompt_text
)
prompt_text = re.sub(
"<video_placeholder>", hf_processor.video_token, prompt_text
)
text_outputs = tokenizer(prompt_text, return_tensors="pt")
return BatchFeature({**text_outputs, **image_outputs, **video_outputs})
processed_data.update(text_outputs)
processed_data.update(image_outputs)
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
+17 -19
View File
@@ -220,27 +220,21 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
"""Basic image-only MultiModalProcessor for InternVL-style models."""
def _call_hf_processor(
def _postprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token_id = hf_processor.ctx_image_token_id
# Since there may be extra tokens in the feature placeholders,
# we need to pass the image token ID to the model to select the
# tokens to merge from the vision encoder outputs
processed_outputs["image_token_id"] = torch.tensor(image_token_id)
processed_data["image_token_id"] = torch.tensor(image_token_id)
return processed_outputs
return processed_data
def _get_image_fields_config(self, hf_inputs: BatchFeature):
image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
@@ -455,19 +449,23 @@ class InternVLMultiModalProcessor(
):
"""InternVL MultiModalProcessor extended for video support"""
def _call_hf_processor(
def _postprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
processed_data = super()._postprocess_hf_mm_data(
mm_data,
hf_processor_mm_kwargs,
processed_data,
)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
if (video_token_id := hf_processor.ctx_video_token_id) is not None:
processed_outputs["video_token_id"] = torch.tensor(video_token_id)
processed_data["video_token_id"] = torch.tensor(video_token_id)
return processed_outputs
return processed_data
def _get_video_fields_config(self, hf_inputs: BatchFeature):
video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
+19 -9
View File
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.pooler import DispatchPooler
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import MultiModalDataItems
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsCrossEncoding, SupportsMultiModal, SupportsScoreTemplate
@@ -54,19 +55,28 @@ class JinaVLScorer(nn.Module):
class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# NOTE: We should reverse the order of the mm_data because the
# query prompt is placed after the document prompt in the score
# template for JinaVLForRanking model, but in mm_data they are
# stored in the opposite order (query first, then document).
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
for _, value in mm_data.items():
value.reverse()
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
@MULTIMODAL_REGISTRY.register_processor(
+15 -9
View File
@@ -461,17 +461,21 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
def media_token_id(self) -> int:
return self.info.get_hf_config().text_config.eos_token_id + 1
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Run the underlying HF processor on text and image data."""
# Text-only input is handled as a special case here.
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data or not mm_data.get("images", []):
prompt_ids = self.info.get_tokenizer().encode(prompt)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
# Images
image_inputs = mm_data.get("images", [])
@@ -493,7 +497,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
tokenizer = self.info.get_tokenizer()
media_token = tokenizer.convert_ids_to_tokens([self.media_token_id])[0]
prompt_replaced = prompt.replace("<image>", media_token)
prompt_replaced = prompt_text.replace("<image>", media_token)
input_ids = tokenizer.encode(prompt_replaced)
input_ids = torch.tensor(input_ids)
@@ -527,7 +531,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
image_token_thw=torch.tensor(image_meta["image_token_thw"]),
pixel_sizes=torch.tensor(pixel_sizes),
)
return BatchFeature(combined_outputs, tensor_type="pt")
processed_data = BatchFeature(combined_outputs, tensor_type="pt")
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates(
self,
+2 -9
View File
@@ -1162,15 +1162,8 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):
class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_keye.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_prompt_updates(
self,
+2 -9
View File
@@ -378,15 +378,8 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_keye.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_prompt_updates(
self,
+18 -8
View File
@@ -37,6 +37,7 @@ from vllm.multimodal.parse import (
DictEmbeddingItems,
ModalityData,
ModalityDataItems,
MultiModalDataItems,
MultiModalDataParser,
)
from vllm.multimodal.processing import (
@@ -226,13 +227,20 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser):
class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]):
"""vLLM multi-modal processor wrapper for Kimi-Audio."""
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Call the HuggingFace processor."""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
# Convert mm_data format: {'audios': [...]} -> {'audio': ...}
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -252,11 +260,13 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
mm_data["audio"] = audio_arrays
# Use the context's call_hf_processor for proper handling
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
mm_kwargs,
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
-10
View File
@@ -258,16 +258,6 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
grid_thws=MultiModalFieldConfig.batched("vision_chunk", keep_on_cpu=True),
)
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path because vision chunk
# is not considered
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
+3
View File
@@ -237,6 +237,9 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+13 -20
View File
@@ -393,45 +393,38 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):
class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
# Text-only input not supported in composite processor
if not (images := mm_data.get("images", [])):
prompt_ids = self.info.get_tokenizer().encode(
prompt, add_special_tokens=False
)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
processed_outputs = super()._call_hf_processor(
prompt,
mm_data,
mm_kwargs,
)
if not mm_data:
return processed_data
images = mm_data.get("images", [])
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [
parsed_images.get_image_size(i) for i in range(len(parsed_images))
]
hf_processor = self.info.get_hf_processor(**mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
num_patches = [
self.info.get_num_patches(
image_width=size.width,
image_height=size.height,
processor=hf_processor,
mm_kwargs=mm_kwargs,
mm_kwargs=hf_processor_mm_kwargs,
)
for size in image_sizes
]
processed_outputs["num_patches"] = torch.tensor(num_patches)
processed_data["num_patches"] = torch.tensor(num_patches)
return processed_outputs
return processed_data
def _get_mm_fields_config(
self,
+21 -22
View File
@@ -39,20 +39,19 @@ _I = TypeVar("_I", bound=Mistral3ProcessingInfo)
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
# NOTE: LightOnOCR does not use break/end tokens, so we remove them here.
input_ids = processed_outputs.get("input_ids")
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
input_ids = processed_data.get("input_ids")
if input_ids is not None:
processor = self.info.get_hf_processor()
tokenizer = self.info.get_tokenizer()
@@ -67,22 +66,22 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
torch.tensor([break_id, end_id]),
)
processed_outputs["input_ids"] = input_ids[keep_mask].unsqueeze(0)
if "attention_mask" in processed_outputs:
processed_outputs["attention_mask"] = processed_outputs[
"attention_mask"
][keep_mask].unsqueeze(0)
processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0)
if "attention_mask" in processed_data:
processed_data["attention_mask"] = processed_data["attention_mask"][
keep_mask
].unsqueeze(0)
# un-pad pixel_values per-image so caches remain independent.
pixel_values = processed_outputs.get("pixel_values")
pixel_values = processed_data.get("pixel_values")
if pixel_values is not None:
image_sizes = processed_outputs["image_sizes"]
image_sizes = processed_data["image_sizes"]
assert len(pixel_values) == len(image_sizes)
processed_outputs["pixel_values"] = [
processed_data["pixel_values"] = [
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
]
return processed_outputs
return processed_data
def _get_mm_fields_config(
self,
+15 -15
View File
@@ -321,30 +321,30 @@ class PixtralHFProcessingInfo(BaseLlavaProcessingInfo):
class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
pixel_values = processed_outputs.get("pixel_values")
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
pixel_values = processed_data.get("pixel_values")
if pixel_values is not None:
# Avoid padding since we need the output for each image to be
# independent of other images for the cache to work correctly
image_sizes = processed_outputs["image_sizes"]
image_sizes = processed_data["image_sizes"]
assert len(pixel_values) == len(image_sizes)
processed_outputs["pixel_values"] = [
processed_data["pixel_values"] = [
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
]
return processed_outputs
return processed_data
def _get_mm_fields_config(
self,
+3
View File
@@ -211,6 +211,9 @@ class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]):
class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+31 -24
View File
@@ -321,22 +321,33 @@ class LlavaOnevisionMultiModalProcessor(
pixel_values_videos=MultiModalFieldConfig.batched("video"),
)
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data)
videos = mm_data.pop("videos", [])
assert isinstance(videos, list)
if not videos:
return super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
# LLaVA-OneVision processor doesn't support multiple videos
# with different sizes when converting back to tensors
@@ -346,19 +357,13 @@ class LlavaOnevisionMultiModalProcessor(
image_token = processor.image_token
video_token = processor.video_token
text_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data={},
mm_kwargs=mm_kwargs,
)
images = mm_data.pop("images", [])
assert isinstance(images, list)
if images:
processor_outputs = super()._call_hf_processor(
prompt=image_token * len(images),
mm_data={"images": images},
mm_kwargs=mm_kwargs,
processor_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=image_token * len(images), **{"images": images}),
hf_processor_mm_kwargs,
)
image_outputs = {
k: v
@@ -370,10 +375,10 @@ class LlavaOnevisionMultiModalProcessor(
pixel_values_videos = []
for video in videos:
item_outputs = super()._call_hf_processor(
prompt=video_token,
mm_data={"videos": video},
mm_kwargs=mm_kwargs,
item_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=video_token, **{"videos": video}),
hf_processor_mm_kwargs,
)
pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
@@ -381,11 +386,13 @@ class LlavaOnevisionMultiModalProcessor(
video_outputs = {"pixel_values_videos": pixel_values_videos}
combined_outputs = dict(
text_outputs,
input_ids=prompt_text,
**image_outputs,
**video_outputs,
)
return BatchFeature(combined_outputs)
processed_data = BatchFeature(combined_outputs)
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates(
self,
+48 -33
View File
@@ -286,7 +286,7 @@ def _validate_video_sources(paths, model_config) -> list[str]:
# participates in the standard decode-stage pipeline.
#
# * codec backend: NOT a loader. OV2's codec path needs the video path string
# to survive into ``_call_hf_processor``, where the HF processor builds the
# to survive into ``_apply_hf_processor_main``, where the HF processor builds the
# codec canvas + smart_resize + patchify
# (pixel_values/image_grid_thw/patch_positions). That transform is
# path-level and inseparable; it cannot be reconstructed from pre-decoded
@@ -302,7 +302,7 @@ def prepare_codec_video_input(video_path: str) -> tuple:
Returns ``(dummy_ndarray, metadata)`` where the ndarray satisfies the
parser's 4-D shape check and the metadata carries the actual path to
our ``_call_hf_processor``. Use as::
our ``_apply_hf_processor_main``. Use as::
multi_modal_data = {"video": prepare_codec_video_input("foo.mp4")}
@@ -1283,7 +1283,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
# ``video_needs_metadata=True`` makes the parser preserve both the
# ``(frames, metadata)`` tuples from the frame backend and the
# ``(dummy, {marker: path})`` tuples from prepare_codec_video_input;
# both are dispatched by metadata content in ``_call_hf_processor``.
# both are dispatched by metadata content in ``_apply_hf_processor_main``.
return LlavaOnevision2MultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
@@ -1547,18 +1547,23 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
class LlavaOnevision2MultiModalProcessor(
BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]
):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# The wrapped OV2 processor is a bare custom class without the standard
# ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path
# fails; overriding this method routes the base class to call us
# directly.
hf_processor = self.info.get_hf_processor(**mm_kwargs)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
merged_kwargs.setdefault("return_tensors", "pt")
call_kwargs = {
k: v
@@ -1627,21 +1632,25 @@ class LlavaOnevision2MultiModalProcessor(
if len(codec_video_paths) > 1
else codec_video_paths[0]
)
# Route through the base ``_call_hf_processor`` so float-tensor
# Route through ``ctx.call_hf_processor`` so float-tensor
# dtype postprocessing is applied automatically; inject
# ``video_backend="codec"`` via mm_kwargs so the wrapped processor
# dispatches to its codec branch.
output = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs={**mm_kwargs, "video_backend": "codec"},
output = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(
**{**hf_processor_mm_kwargs, "video_backend": "codec"}
),
dict(text=prompt_text, **mm_data),
{**hf_processor_mm_kwargs, "video_backend": "codec"},
)
data = dict(output)
return BatchFeature(
processed_data = BatchFeature(
self._rename_codec_outputs_to_video(
data, codec_video_paths, hf_processor
)
)
processed_data.update(passthrough_data)
return processed_data
# ---- Frame backend (registered LlavaOnevision2VideoBackend) ------
# Every non-codec video reaches here as a ``(frames_ndarray, metadata)``
@@ -1657,7 +1666,9 @@ class LlavaOnevision2MultiModalProcessor(
# local-file gating is enforced by the connector before decoding.
if videos_present:
timestamp_decimals = int(
mm_kwargs.get("timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS)
hf_processor_mm_kwargs.get(
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
)
)
per_video_frames: list[list[Image.Image]] = []
@@ -1671,7 +1682,7 @@ class LlavaOnevision2MultiModalProcessor(
# ``<{t} seconds><|vision_start|><|image_pad|><|vision_end|>``
# blocks (matches the OV2 hf-chat reference exactly).
new_prompt = _expand_video_markers_in_prompt(
prompt,
prompt_text,
per_video_timestamps,
timestamp_decimals=timestamp_decimals,
)
@@ -1700,7 +1711,7 @@ class LlavaOnevision2MultiModalProcessor(
row_is_video: list[bool] = []
vid_idx = 0
img_idx = 0
for marker in marker_pattern.finditer(prompt):
for marker in marker_pattern.finditer(prompt_text):
if marker.lastgroup == "video":
frames = per_video_frames[vid_idx]
vid_idx += 1
@@ -1723,15 +1734,15 @@ class LlavaOnevision2MultiModalProcessor(
merged_mm_data.pop("videos", None)
merged_mm_data["images"] = flat_frames
# Route through the base ``_call_hf_processor`` (applies float-tensor
# Route through ``ctx.call_hf_processor`` (applies float-tensor
# dtype postprocessing automatically). The wrapped processor's image
# branch ignores video/codec-only kwargs and does not forward extra
# **kwargs to the image processor, so passing the full merged kwarg
# set here is a no-op beyond return_tensors/padding.
output = super()._call_hf_processor(
prompt=new_prompt,
mm_data=merged_mm_data,
mm_kwargs=mm_kwargs,
output = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=new_prompt, **merged_mm_data),
hf_processor_mm_kwargs,
)
data = dict(output)
@@ -1788,17 +1799,21 @@ class LlavaOnevision2MultiModalProcessor(
(len(per_video_timestamps),), dtype=torch.long
)
return BatchFeature(data)
processed_data = BatchFeature(data)
processed_data.update(passthrough_data)
return processed_data
# ---- Image-only / text-only call --------------------------------
# No videos present: delegate to the base ``_call_hf_processor``, which
# No videos present: delegate to ``ctx.call_hf_processor``, which
# runs the wrapped processor over the (possibly empty) image set and
# applies float-tensor dtype postprocessing automatically.
return super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
def _rename_codec_outputs_to_video(
self,
+9 -20
View File
@@ -26,7 +26,7 @@
import collections
import collections.abc
from collections.abc import Callable, Iterable, Mapping, Sequence
from typing import Annotated, Any, TypeAlias, cast
from typing import Annotated, TypeAlias, cast
import numpy as np
import torch
@@ -583,15 +583,17 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
class MiDashengLMMultiModalProcessor(
BaseMultiModalProcessor[MiDashengLMProcessingInfo]
):
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any],
) -> BatchFeature:
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
# + Padding
min_audio_len = self.info.get_min_audio_len()
processed_audios = [
np.pad(
@@ -608,20 +610,7 @@ class MiDashengLMMultiModalProcessor(
if processed_audios:
mm_data["audio"] = processed_audios
if not mm_data.get("audio", []):
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
mm_kwargs = dict(
**mm_kwargs,
)
return super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
return mm_data, hf_processor_mm_kwargs
def _get_mm_fields_config(
self,
+18 -5
View File
@@ -912,16 +912,23 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")
return fields
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.
Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses
the singular form.
"""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
# Remap audios → audio (MiMoOmniProcessor uses singular param name)
if "audios" in mm_data:
mm_data = {**mm_data, "audio": mm_data["audios"]}
@@ -999,7 +1006,13 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
mm_data = {**mm_data, "videos": converted}
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
mm_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates(
self,
+1 -1
View File
@@ -448,7 +448,7 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems):
audio_inputs = {}
else:
audio_inputs = self._base_call_hf_processor(
audio_inputs = self._call_hf_processor_on_prompts(
prompts=[self.info.audio_pattern] * len(parsed_audios),
mm_data={"audios": [[audio] for audio in parsed_audios]},
mm_kwargs={**mm_kwargs, "chunk_input": True},
+26 -18
View File
@@ -881,7 +881,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
image_inputs = {}
else:
image_inputs = self._base_call_hf_processor(
image_inputs = self._call_hf_processor_on_prompts(
prompts=[self.info.image_pattern] * len(parsed_images),
mm_data={"images": [[image] for image in parsed_images]},
mm_kwargs=mm_kwargs,
@@ -906,7 +906,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
video_inputs = {}
else:
video_inputs = self._base_call_hf_processor(
video_inputs = self._call_hf_processor_on_prompts(
prompts=[
self.info.image_pattern * len(video) for video in parsed_videos
],
@@ -983,7 +983,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
return new_token_ids, placeholders
def _base_call_hf_processor(
def _call_hf_processor_on_prompts(
self,
prompts: list[str],
mm_data: Mapping[str, Sequence[object]],
@@ -993,19 +993,19 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
) -> dict[str, NestedTensors]:
# This processor supports zipping prompt and mm_data together
if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}:
inputs = super()._call_hf_processor(
prompt=prompts, # type: ignore
mm_data=mm_data,
mm_kwargs=mm_kwargs,
inputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompts, **mm_data),
mm_kwargs,
)
else:
inputs = defaultdict[str, list[torch.Tensor]](list)
for i, prompt in enumerate(prompts):
inputs_one = super()._call_hf_processor(
prompt=prompt,
mm_data={k: v[i] for k, v in mm_data.items()},
mm_kwargs=mm_kwargs,
inputs_one = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **{k: v[i] for k, v in mm_data.items()}),
mm_kwargs,
)
for k, v in inputs_one.items():
@@ -1014,23 +1014,31 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
return {k: inputs[k] for k in out_keys}
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
tokenizer = self.info.get_tokenizer()
input_ids = torch.tensor([tokenizer.encode(prompt)])
mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs)
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
return BatchFeature(
processed_data = BatchFeature(
{
"input_ids": input_ids,
**mm_inputs,
}
)
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates(
self,
+15 -15
View File
@@ -233,30 +233,30 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo])
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
pixel_values = processed_outputs.get("pixel_values")
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
pixel_values = processed_data.get("pixel_values")
if pixel_values is not None:
# Avoid padding since we need the output for each image to be
# independent of other images for the cache to work correctly
image_sizes = processed_outputs["image_sizes"]
image_sizes = processed_data["image_sizes"]
assert len(pixel_values) == len(image_sizes)
processed_outputs["pixel_values"] = [
processed_data["pixel_values"] = [
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
]
return processed_outputs
return processed_data
def _get_mm_fields_config(
self,
+16 -16
View File
@@ -589,23 +589,23 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
processor = self.info.get_hf_processor(**mm_kwargs)
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_processor = processor.image_processor
vision_config = self.info.get_hf_config().vision_config
if processed_outputs.get("pixel_values") is not None:
if processed_data.get("pixel_values") is not None:
assert "images" in mm_data, (
"images expected to be in mm_data when pixel_values is present"
)
@@ -636,10 +636,10 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
1 if r_h * r_w == 1 else 1 + r_h * r_w for (r_h, r_w) in aspect_ratios
]
processed_outputs["aspect_ratios"] = torch.tensor(aspect_ratios)
processed_outputs["patches_per_image"] = torch.tensor(patches_per_image)
processed_data["aspect_ratios"] = torch.tensor(aspect_ratios)
processed_data["patches_per_image"] = torch.tensor(patches_per_image)
return processed_outputs
return processed_data
def _get_mm_fields_config(
self,
+42 -39
View File
@@ -1123,17 +1123,44 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
def _call_hf_processor(
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
processor = self.info.get_hf_processor()
# The chat template is already applied to the prompt tokens
# Use message_format="none" to avoid applying it again
# Prepend an empty space if `always_start_with_space` is True
tokens = processor.get_tokens_input(
self.info.get_tokenizer().decode(prompt),
message_format="none",
always_start_with_space=True,
)
# Prepend a BOS token id to the tokens
return self.info.ctx.call_hf_processor(
processor.process,
dict(tokens=tokens),
)["input_ids"].tolist()
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
hf_processor = self.info.get_hf_processor(**mm_kwargs)
processed_outputs = self.info.ctx.call_hf_processor(
mm_counts = mm_items.get_all_counts()
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not processor_data:
return BatchFeature(dict(passthrough_data))
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
processed_data = self.info.ctx.call_hf_processor(
hf_processor.process,
dict(text=prompt, **mm_data),
mm_kwargs,
dict(text=prompt_text, **processor_data),
hf_processor_mm_kwargs,
)
tokenizer = hf_processor.tokenizer
@@ -1141,17 +1168,16 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
image_processor = hf_processor.image_processor
input_ids: torch.Tensor = processed_outputs.pop("input_ids")
processed_outputs["input_ids"] = input_ids.unsqueeze(0)
processed_data.pop("input_ids")
if (images := mm_data.get("images")) is not None:
if (images := processor_data.get("images")) is not None:
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [
parsed_images.get_image_size(i) for i in range(len(parsed_images))
]
feat_is_patch = processed_outputs["image_input_idx"] >= 0
feat_is_patch = processed_data["image_input_idx"] >= 0
tilings = [
self.info.select_tiling(
@@ -1165,35 +1191,12 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
num_crops = torch.tensor(tilings).prod(-1) + 1
assert num_crops.sum() == len(feat_is_patch)
processed_outputs["num_crops"] = num_crops
processed_outputs["img_patch_id"] = image_patch_id
processed_data["num_crops"] = num_crops
processed_data["img_patch_id"] = image_patch_id
return processed_outputs
processed_data.update(passthrough_data)
def _apply_hf_processor_tokens_only(
self,
prompt_tokens: list[int],
) -> list[int]:
processor = self.info.get_hf_processor()
# The chat template is already applied to the prompt tokens
# Use message_format="none" to avoid applying it again
# Prepend an empty space if `always_start_with_space` is True
tokens = processor.get_tokens_input(
self.info.get_tokenizer().decode(prompt_tokens),
message_format="none",
always_start_with_space=True,
)
# Prepend a BOS token id to the tokens
processed_data = self.info.ctx.call_hf_processor(
processor.process,
dict(tokens=tokens),
)
prompt_ids = processed_data.pop("input_ids").tolist()
print(prompt_ids, len(prompt_ids))
return prompt_ids
return processed_data
def _get_mm_fields_config(
self,
+41 -32
View File
@@ -1929,30 +1929,33 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
def _apply_hf_processor_tokens_only(
self,
prompt_tokens: list[int],
) -> list[int]:
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
processor = self.info.get_hf_processor()
tokenizer = processor.tokenizer
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
if len(prompt_tokens) == 0 or prompt_tokens[0] != bos_token_id:
if len(prompt) == 0 or prompt[0] != bos_token_id:
# Prepend the bos token to the prompt tokens
prompt_tokens = [bos_token_id] + prompt_tokens
prompt = [bos_token_id] + prompt
return prompt_tokens
return prompt
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
mm_counts = mm_items.get_all_counts()
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
mm_data = dict(processor_data)
hf_config = self.info.get_hf_config()
hf_processor = self.info.get_hf_processor(**mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature:
res = hf_processor(text=text, images=images, videos=videos, **kwargs)
@@ -1984,14 +1987,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
# NOTE: metadata.frames_indices indicates
# the sampled frames indices of pre-sampled videos, which is
# used to calculate the timestamps. Make sure that
# do_sample_frames in mm_kwargs is false for presampled videos.
# do_sample_frames in hf_processor_mm_kwargs is false for
# presampled videos.
# NOTE: a copy of mm_kwargs is created to update do_sample_frames,
# otherwise mm_hash for the object will be incorrect.
video_mm_kwargs = dict(**mm_kwargs)
# NOTE: a copy of hf_processor_mm_kwargs is created to update
# do_sample_frames, otherwise mm_hash for the object will be
# incorrect.
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
if "do_sample_frames" not in video_mm_kwargs:
# molmo_utils already has "do_sample_frames" in
# mm_kwargs, don't overwrite it.
# hf_processor_mm_kwargs, don't overwrite it.
video_mm_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", False
)
@@ -2015,7 +2020,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
input_ids = input_ids[:, 1:]
video_string = tokenizer.batch_decode(input_ids)[0]
prompt = prompt.replace(VIDEO_PROMPT, video_string, 1)
prompt_text = prompt_text.replace(VIDEO_PROMPT, video_string, 1)
video_grids = video_outputs.pop("video_grids")
assert video_grids[:, 0].sum() == len(
@@ -2054,10 +2059,10 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
else:
all_video_outputs = dict()
processed_outputs = self.info.ctx.call_hf_processor(
processed_data = self.info.ctx.call_hf_processor(
patched_call,
dict(text=prompt, **mm_data),
mm_kwargs,
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
if (images := mm_data.get("images")) is not None:
@@ -2077,22 +2082,22 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
for image_size in image_sizes
]
num_crops = torch.tensor(tilings).prod(-1) + 1
assert sum(num_crops) == len(processed_outputs["pixel_values"])
assert sum(num_crops) == processed_outputs["image_num_crops"].sum().item()
assert sum(num_crops) == len(processed_data["pixel_values"])
assert sum(num_crops) == processed_data["image_num_crops"].sum().item()
image_grids = processed_outputs.pop("image_grids")
image_grids = processed_data.pop("image_grids")
image_num_pooled_patches = image_grids[:, :2].prod(dim=1) + image_grids[
:, 2:
].prod(dim=1)
processed_outputs["image_num_pooled_patches"] = image_num_pooled_patches
n_patches = processed_outputs["pixel_values"].shape[1]
processed_outputs["image_num_patches"] = (
processed_outputs["image_num_crops"] * n_patches
processed_data["image_num_pooled_patches"] = image_num_pooled_patches
n_patches = processed_data["pixel_values"].shape[1]
processed_data["image_num_patches"] = (
processed_data["image_num_crops"] * n_patches
)
(
processed_outputs["image_tokens"],
processed_outputs["num_image_tokens"],
processed_data["image_tokens"],
processed_data["num_image_tokens"],
) = build_flat_image_bool_length(
image_grids,
hf_config,
@@ -2101,7 +2106,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
use_single_crop_start_token=hf_processor.use_single_crop_start_token,
)
return BatchFeature({**processed_outputs, **all_video_outputs})
processed_data.update(all_video_outputs)
processed_data.update(passthrough_data)
processed_data.pop("input_ids")
return processed_data
def _get_mm_fields_config(
self,
-10
View File
@@ -942,16 +942,6 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
image_placeholder: str = "<image>"
bos_image_placeholder: str = "<|endoftext|><image>"
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Moondream3's processor handles images directly rather than exposing a
# separate `image_processor`, so keep the cache path on text+MM calls.
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
@cached_property
def bos_image_placeholder_tokens(self) -> list[int]:
tokenizer = self.info.get_tokenizer()
+17 -8
View File
@@ -1335,23 +1335,32 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo
class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
mm_data["audio"] = audios
mm_kwargs = dict(mm_kwargs)
processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs)
return self.info.ctx.call_hf_processor(
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
processor_kwargs = _filter_moss_audio_processor_config(hf_processor_mm_kwargs)
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**processor_kwargs),
dict(text=prompt, **mm_data),
mm_data,
{},
)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
@@ -461,24 +461,29 @@ class MossTranscribeDiarizeDummyInputsBuilder(
class MossTranscribeDiarizeMultiModalProcessor(
BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]
):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
audios = _get_audios_from_mm_data(mm_data)
if not audios:
input_ids = tokenizer.encode(prompt, add_special_tokens=False)
return BatchFeature({"input_ids": [input_ids]}, tensor_type="pt")
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
processed = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, audio=audios),
mm_kwargs,
)
return _add_vllm_audio_metadata(processed, len(audios))
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
audios = _get_audios_from_mm_data(mm_data)
return dict(audio=audios), hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
audios = _get_audios_from_mm_data(mm_data)
if audios:
processed_data = _add_vllm_audio_metadata(processed_data, len(audios))
return processed_data
def _get_mm_fields_config(
self,
+15 -7
View File
@@ -262,13 +262,19 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
class MuseGlimmerMultiModalProcessor(
BaseMultiModalProcessor[MuseGlimmerProcessingInfo]
):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processor = self.info.get_hf_processor(**mm_kwargs)
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = processor.tokenizer
config = self.info.get_hf_config()
images = mm_data.get("images", ())
@@ -276,7 +282,7 @@ class MuseGlimmerMultiModalProcessor(
if not isinstance(images, Sequence) or not isinstance(videos, Sequence):
raise TypeError("MuseGlimmer multi-modal data must be a sequence")
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
prompt_ids = tokenizer.encode(prompt_text, add_special_tokens=False)
image_sentinel_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)
video_sentinel_id = tokenizer.convert_tokens_to_ids(VIDEO_TOKEN)
if prompt_ids.count(image_sentinel_id) != len(images):
@@ -354,7 +360,9 @@ class MuseGlimmerMultiModalProcessor(
video_pixel_values=video_pixels,
video_feature_sizes=torch.tensor(video_sizes),
)
return BatchFeature(data=data, tensor_type=None)
processed_data = BatchFeature(data=data, tensor_type=None)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
+8 -19
View File
@@ -361,18 +361,8 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
class NanoNemotronVLMultiModalProcessor(
BaseMultiModalProcessor[NanoNemotronVLProcessingInfo]
):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Bypass `call_hf_processor_mm_only` by no-op overriding`_call_hf_processor`,
so it chooses this path:
`type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor`
"""
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_image_fields_config(self, hf_inputs: BatchFeature):
if self.info.is_dynamic_tiler:
@@ -406,7 +396,9 @@ class NanoNemotronVLMultiModalProcessor(
)
def _get_audio_fields_config(self, hf_inputs: BatchFeature):
audio_num_clips = torch.as_tensor(hf_inputs["audio_num_clips"])
audio_num_clips = torch.as_tensor(
hf_inputs.get("audio_num_clips", torch.empty(0))
)
return dict(
input_audio_features=MultiModalFieldConfig.flat_from_sizes(
@@ -733,11 +725,8 @@ class NanoNemotronVLMultiModalProcessor(
# Bypass the cached path: the HF processor must receive the
# prompt (with injected <so_embedding>) and the audio data
# together so it can perform audio-token replacement natively.
(
prompt_ids,
mm_info,
is_update_applied,
) = self._apply_hf_processor(inputs, timing_ctx)
mm_info = self._apply_hf_processor(inputs, timing_ctx)
prompt_ids = self._postprocess_prompt(inputs.prompt)
with timing_ctx.record("apply_prompt_updates"):
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
@@ -745,7 +734,7 @@ class NanoNemotronVLMultiModalProcessor(
prompt_ids=prompt_ids,
mm_kwargs=mm_info.kwargs,
mm_prompt_updates=mm_info.prompt_updates,
is_update_applied=is_update_applied,
is_update_applied=self.hf_processor_applies_updates,
)
mm_placeholder_ranges = {
@@ -411,22 +411,6 @@ class NemotronParseMultiModalProcessor(
) -> list[int]:
return [0]
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
else:
hf_processor = self.info.get_hf_processor()
tokenizer = hf_processor.tokenizer
processed_outputs = tokenizer(
prompt, add_special_tokens=False, return_tensors="pt"
)
return processed_outputs
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+10 -21
View File
@@ -341,41 +341,30 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
# -300 is image_atom token, filter them out
return [vte_vocab_size + x + 300 for x in image_indicators if x < -300]
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
# Avoid warning from HF logger for text-only input
tokenizer = self.info.get_tokenizer()
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
return processed_data
hf_processor = self.info.get_hf_processor()
image_indicators = [
hf_processor.construct_image_indicators(grid)
for grid in processed_outputs["grids"]
for grid in processed_data["grids"]
]
indicator_tokens = [
self.image_indicators_to_visual_tokens(indicator)
for indicator in image_indicators
]
processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens)
return processed_outputs
processed_data["indicator_tokens"] = torch.tensor(indicator_tokens)
def _apply_hf_processor_tokens_only(
self,
prompt_tokens: list[int],
) -> list[int]:
return prompt_tokens
return processed_data
def _get_mm_fields_config(
self,
+12 -22
View File
@@ -332,53 +332,43 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
if x >= INDICATOR_IDS[0]
]
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
# Avoid warning from HF logger for text-only input
tokenizer = self.info.get_tokenizer()
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
return processed_data
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
hf_processor = self.info.get_hf_processor()
if "videos" in mm_data:
visual_indicators = [
hf_processor.construct_visual_indicators((1, 1, 1), True)
for grid in processed_outputs["video_grids"]
for grid in processed_data["video_grids"]
]
indicator_tokens = [
self.visual_indicators_to_visual_tokens(indicator)
for indicator in visual_indicators
]
processed_outputs["video_indicator_tokens"] = torch.tensor(indicator_tokens)
processed_data["video_indicator_tokens"] = torch.tensor(indicator_tokens)
if "images" in mm_data:
visual_indicators = [
hf_processor.construct_visual_indicators((1, 1, 1), False)
for grid in processed_outputs["grids"]
for grid in processed_data["grids"]
]
indicator_tokens = [
self.visual_indicators_to_visual_tokens(indicator)
for indicator in visual_indicators
]
processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens)
return processed_outputs
processed_data["indicator_tokens"] = torch.tensor(indicator_tokens)
def _apply_hf_processor_tokens_only(
self,
prompt_tokens: list[int],
) -> list[int]:
return prompt_tokens
return processed_data
def _get_mm_fields_config(
self,
+29 -24
View File
@@ -244,32 +244,37 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing
class PaddleOCRVLMultiModalProcessor(
BaseMultiModalProcessor[PaddleOCRVLProcessingInfo]
):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
final_mm_kwargs = dict(mm_kwargs or {})
final_mm_kwargs.setdefault("images_kwargs", {})
# vLLM use PIL.Image, always set channel_last
final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
processed_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**final_mm_kwargs),
dict(text=prompt, **mm_data),
mm_kwargs,
)
num_patches_per_image = processed_outputs["image_grid_thw"].prod(-1)
processed_outputs["pixel_values"] = processed_outputs["pixel_values"].split(
num_patches_per_image.tolist()
)
else:
tokenizer = self.info.get_tokenizer()
processed_outputs = tokenizer(
prompt, add_special_tokens=True, return_tensors="pt"
)
return processed_outputs
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
final_mm_kwargs = dict(hf_processor_mm_kwargs or {})
final_mm_kwargs.setdefault("images_kwargs", {})
# vLLM use PIL.Image, always set channel_last
final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**final_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
num_patches_per_image = processed_data["image_grid_thw"].prod(-1)
processed_data["pixel_values"] = processed_data["pixel_values"].split(
num_patches_per_image.tolist()
)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
-17
View File
@@ -152,23 +152,6 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo
class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
if not mm_data:
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
return super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+13 -13
View File
@@ -396,19 +396,19 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
input_ids = processed_outputs["input_ids"]
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not mm_data:
return processed_data
input_ids = processed_data["input_ids"]
assert isinstance(input_ids, torch.Tensor)
# Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
@@ -416,7 +416,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
# Therefore, we need to do an early replacement here
input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID)
return processed_outputs
return processed_data
def _get_mm_fields_config(
self,
+25 -16
View File
@@ -868,43 +868,52 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
sr = self.info.get_feature_extractor(**mm_kwargs).sampling_rate
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
sr = self.info.get_feature_extractor(**hf_processor_mm_kwargs).sampling_rate
if audio_data := mm_data.get("audios", []):
mm_data["audios"] = [(data, sr) for data in audio_data]
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
num_img_tokens = [
self.info.get_num_image_tokens(
image_width=img_size[0],
image_height=img_size[1],
processor=hf_processor,
)
for img_size in processed_outputs["image_sizes"]
for img_size in processed_data["image_sizes"]
]
processed_outputs["num_img_tokens"] = num_img_tokens
processed_data["num_img_tokens"] = num_img_tokens
audio_features = processed_outputs["input_audio_embeds"]
audio_features = processed_data["input_audio_embeds"]
feature_sizes = [
self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data
]
processed_outputs["input_audio_embeds"] = [
processed_data["input_audio_embeds"] = [
audio_features[idx, :size] for idx, size in enumerate(feature_sizes)
]
processed_data.update(passthrough_data)
return processed_outputs
return processed_data
def _get_mm_fields_config(
self,
+18 -10
View File
@@ -142,16 +142,22 @@ class Phi4SiglipDummyInputsBuilder(
class Phi4SiglipMultiModalProcessor(
BaseMultiModalProcessor[Phi4SiglipProcessingInfo],
):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
processed = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
processed = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
# The HF processor's tokenizer_image_token() replaces the "<image>"
@@ -163,10 +169,12 @@ class Phi4SiglipMultiModalProcessor(
# NOTE: tokenizer.__call__() (not .encode()) must be used so that
# added/special tokens like <|user|>, <|end|> are kept as single IDs.
tokenizer = self.info.get_tokenizer()
new_ids = tokenizer(prompt).input_ids
new_ids = tokenizer(prompt_text).input_ids
processed["input_ids"] = torch.tensor([new_ids])
return processed
processed_data = processed
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
+22 -20
View File
@@ -51,12 +51,10 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
MultiModalProcessingInfo,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
TimingContext,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
@@ -213,6 +211,9 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]):
# The tokens are already inserted by the chat template
hf_processor_applies_updates = True
def _get_mm_fields_config(
self,
hf_inputs: Mapping[str, NestedTensors],
@@ -220,24 +221,35 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
) -> Mapping[str, MultiModalFieldConfig]:
return dict(images=MultiModalFieldConfig.batched("image"))
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
# Avoid padding issue
dict(**mm_kwargs, return_tensors=None),
dict(**hf_processor_mm_kwargs, return_tensors=None),
)
# Missing batch dimension
if is_list_of(outputs["input_ids"], int):
outputs["input_ids"] = [outputs["input_ids"]]
return outputs
processed_data = outputs
processed_data.update(passthrough_data)
return processed_data
def _get_prompt_updates(
self,
@@ -273,16 +285,6 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
),
]
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx)
# NOTE: The tokens are already inserted by the chat template
return prompt_ids, mm_info, True
@MULTIMODAL_REGISTRY.register_processor(
PixtralMultiModalProcessor,
@@ -499,83 +499,6 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
class Qwen2_5OmniThinkerMultiModalProcessor(
BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
if audios:
# NOTE: Qwen2.5-Omni processor accept "audio"
mm_data["audio"] = audios
mm_kwargs = dict(
**mm_kwargs,
)
elif (
mm_kwargs.get("use_audio_in_video")
and "audio" not in mm_data
and mm_data.get("videos")
):
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
# and populated mm_data["audio"] itself before delegating here,
# in which case this isn't a genuine "no audio" case. Likewise,
# mm_data can be empty on a multimodal-processor-cache hit, where
# there's nothing to (re-)process this call and the real result
# comes from the cache — not a genuine "no audio" case either.
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
if mm_data.get("videos") and (
merged.keys() & {"size", "min_pixels", "max_pixels"}
):
mm_kwargs = dict(mm_kwargs)
video_size = dict(self.info.get_hf_processor().video_processor.size)
size_override = merged.get("size")
if size_override is not None:
video_size = video_size | size_override
min_pixels = merged.get("min_pixels")
if min_pixels is not None:
video_size["shortest_edge"] = min_pixels
max_pixels = merged.get("max_pixels")
if max_pixels is not None:
video_size["longest_edge"] = max_pixels
mm_kwargs["size"] = video_size
hf_inputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
input_features = hf_inputs.pop("input_features", None)
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
if "input_audio_features" not in hf_inputs and input_features is not None:
if feature_attention_mask is not None:
input_features = input_features.permute(0, 2, 1)[
feature_attention_mask.bool()
].permute(1, 0)
hf_inputs["input_audio_features"] = input_features
if (
"audio_feature_lengths" not in hf_inputs
and feature_attention_mask is not None
):
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
if video_second_per_grid is not None:
hf_inputs["second_per_grid_ts"] = video_second_per_grid
use_audio_in_video = mm_kwargs.get("use_audio_in_video", False)
hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
return hf_inputs
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -885,7 +808,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
),
]
def _apply_hf_processor_mm_only(
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
@@ -903,13 +826,87 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
)
mm_counts["audio"] -= mm_counts["video"]
_, mm_processed_data = self._apply_hf_processor_text_mm(
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
mm_data = dict(processor_data)
audios = mm_data.pop("audios", [])
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
if audios:
# NOTE: Qwen2.5-Omni processor accept "audio"
mm_data["audio"] = audios
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
)
elif (
hf_processor_mm_kwargs.get("use_audio_in_video")
and "audio" not in mm_data
and mm_data.get("videos")
):
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
# and populated mm_data["audio"] itself before delegating here,
# in which case this isn't a genuine "no audio" case. Likewise,
# mm_data can be empty on a multimodal-processor-cache hit, where
# there's nothing to (re-)process this call and the real result
# comes from the cache — not a genuine "no audio" case either.
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
if mm_data.get("videos") and (
merged.keys() & {"size", "min_pixels", "max_pixels"}
):
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
video_size = dict(self.info.get_hf_processor().video_processor.size)
size_override = merged.get("size")
if size_override is not None:
video_size = video_size | size_override
min_pixels = merged.get("min_pixels")
if min_pixels is not None:
video_size["shortest_edge"] = min_pixels
max_pixels = merged.get("max_pixels")
if max_pixels is not None:
video_size["longest_edge"] = max_pixels
hf_processor_mm_kwargs["size"] = video_size
hf_inputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
return mm_processed_data
input_features = hf_inputs.pop("input_features", None)
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
if "input_audio_features" not in hf_inputs and input_features is not None:
if feature_attention_mask is not None:
input_features = input_features.permute(0, 2, 1)[
feature_attention_mask.bool()
].permute(1, 0)
hf_inputs["input_audio_features"] = input_features
if (
"audio_feature_lengths" not in hf_inputs
and feature_attention_mask is not None
):
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
if video_second_per_grid is not None:
hf_inputs["second_per_grid_ts"] = video_second_per_grid
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
processed_data = hf_inputs
processed_data.update(passthrough_data)
processed_data.pop("input_ids")
return processed_data
class Qwen2_5OmniConditionalGenerationMixin:
-10
View File
@@ -1154,16 +1154,6 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
)
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_qwen2_5_vl.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
+12 -21
View File
@@ -235,36 +235,27 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any],
) -> BatchFeature:
# NOTE - we rename audios -> audio in mm data because transformers has
# deprecated audios for the qwen2audio processor and will remove
# support for it in transformers 4.54.
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
mm_data["audio"] = audios
# Text-only input not supported in composite processor
if not mm_data.get("audio", []):
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_kwargs = dict(
**mm_kwargs,
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
return super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
)
return mm_data, hf_processor_mm_kwargs
def _get_mm_fields_config(
self,
@@ -1195,79 +1195,95 @@ Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder
class Qwen3OmniMoeThinkerMultiModalProcessor(
Qwen2_5OmniThinkerMultiModalProcessor,
):
def _call_hf_processor(
def _get_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
def pad_to_hop_length(x: np.ndarray, hop_length: int) -> np.ndarray:
length = x.shape[-1]
if length % hop_length != 0:
pad_length = hop_length - (length % hop_length)
x = np.pad(x, (0, pad_length), mode="constant", constant_values=0)
return x
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
hop_length = feature_extractor.hop_length
audios = processor_data.get("audios")
if audios:
# NOTE: Qwen3-Omni processor accept "audio"
feature_extractor = self.info.get_feature_extractor()
hop_length = feature_extractor.hop_length
def pad_to_hop_length(x: np.ndarray) -> np.ndarray:
length = x.shape[-1]
if length % hop_length != 0:
pad_length = hop_length - (length % hop_length)
x = np.pad(x, (0, pad_length), mode="constant", constant_values=0)
return x
# To make sure the cache works with padding=True, we pre-padded
# the audio to multiple of hop_length.
mm_data["audio"] = [
pad_to_hop_length(audio, hop_length)
# the audio to a multiple of hop_length.
processor_data = dict(processor_data)
processor_data["audios"] = [
pad_to_hop_length(audio)
if isinstance(audio, np.ndarray)
else (pad_to_hop_length(audio[0], hop_length), audio[1])
else (pad_to_hop_length(audio[0]), audio[1])
for audio in audios
]
return processor_data, passthrough_data
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_items.get_all_counts().get("audio", 0):
# TODO(Isotr0py): Remove this patch after upstream fix PR
# released and Transformers version update:
# https://github.com/huggingface/transformers/pull/41473
mm_kwargs = dict(mm_kwargs)
mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {})
mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {})
elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"):
# mm_data can be empty on a multimodal-processor-cache hit, where
# there's nothing to (re-)process this call and the real result
# comes from the cache — not a genuine "no audio" case.
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
hf_processor_mm_kwargs["audio_kwargs"] = dict(
hf_processor_mm_kwargs.get("audio_kwargs") or {}
)
hf_processor_mm_kwargs["text_kwargs"] = dict(
hf_processor_mm_kwargs.get("text_kwargs") or {}
)
hf_inputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
processed_data = super()._apply_hf_processor_main(
mm_items,
hf_processor_mm_kwargs,
)
if (
"audio_feature_lengths" in hf_inputs
and "feature_attention_mask" in hf_inputs
and (audios := mm_data.get("audio", []))
"audio_feature_lengths" in processed_data
and "feature_attention_mask" in processed_data
):
audio_num_frames = []
for _, audio in enumerate(audios):
audio_length = len(audio[0]) if isinstance(audio, tuple) else len(audio)
num_frame = (
(audio_length // hop_length)
if audio_length % hop_length == 0
else (audio_length // hop_length - 1)
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, _ = self._get_hf_mm_data(valid_mm_items)
audios = processor_data.get("audios", [])
if audios:
feature_extractor = self.info.get_feature_extractor(
**hf_processor_mm_kwargs
)
if mm_kwargs.get("truncation", False):
num_frame = min(
num_frame, feature_extractor.n_samples // hop_length
hop_length = feature_extractor.hop_length
audio_num_frames = []
for audio in audios:
audio_length = (
len(audio[0]) if isinstance(audio, tuple) else len(audio)
)
audio_num_frames.append(num_frame)
hf_inputs["feature_attention_mask"] = [
torch.ones(num_frame) for num_frame in audio_num_frames
]
hf_inputs["audio_feature_lengths"] = torch.tensor(audio_num_frames)
return hf_inputs
num_frame = (
(audio_length // hop_length)
if audio_length % hop_length == 0
else (audio_length // hop_length - 1)
)
if hf_processor_mm_kwargs.get("truncation", False):
num_frame = min(
num_frame, feature_extractor.n_samples // hop_length
)
audio_num_frames.append(num_frame)
processed_data["feature_attention_mask"] = [
torch.ones(num_frame) for num_frame in audio_num_frames
]
processed_data["audio_feature_lengths"] = torch.tensor(audio_num_frames)
return processed_data
def _maybe_apply_prompt_updates(
self,
+35 -22
View File
@@ -1280,12 +1280,21 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
return proc_impl is not None and proc_impl is not mixin_impl
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data)
# Separate video processing from image processing. Because the videos
@@ -1314,8 +1323,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# NOTE: a copy of is created to update do_sample_frames,
# otherwise mm_hash for the object will be incorrect.
video_mm_kwargs = dict(**mm_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
video_size = dict(self.info.get_video_processor().size)
size_override = merged.get("size")
@@ -1365,10 +1374,13 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
video_mm_kwargs["fps"] = None
video_outputs = super()._call_hf_processor(
prompt="<|vision_start|><|video_pad|><|vision_end|>",
mm_data=video_mm_data,
mm_kwargs=video_mm_kwargs,
video_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**video_mm_kwargs),
dict(
text="<|vision_start|><|video_pad|><|vision_end|>",
**video_mm_data,
),
video_mm_kwargs,
)
# Discard HF output input_ids — we use get_video_repl below
@@ -1424,12 +1436,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# fps/num_frames are video-only kwargs already consumed by the loop;
# exclude them so the text/image processor call below never gets a list.
non_video_mm_kwargs = {
k: v for k, v in mm_kwargs.items() if k not in ("fps", "num_frames")
k: v
for k, v in hf_processor_mm_kwargs.items()
if k not in ("fps", "num_frames")
}
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=non_video_mm_kwargs,
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**non_video_mm_kwargs),
dict(text=prompt_text, **mm_data),
non_video_mm_kwargs,
)
# Replace each placeholder with pre-computed video tokens.
@@ -1443,20 +1457,19 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
hf_config.video_token_id,
hf_config.vision_end_token_id,
]
input_ids = processed_outputs.pop("input_ids")
input_ids = processed_data.pop("input_ids")
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
expanded_ids = _replace_video_token_placeholders(
prompt_ids, video_target, video_input_ids_lst
)
processed_outputs["input_ids"] = [expanded_ids]
processed_data["input_ids"] = [expanded_ids]
combined_outputs = dict(
processed_outputs,
**video_outputs,
)
return BatchFeature(combined_outputs)
processed_data.update(video_outputs)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,
@@ -345,28 +345,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
return mm_fields
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Run the HF processor and unpad inputs."""
try:
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
except ValueError:
if any(mm_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
prompt_ids = self.info.get_tokenizer().encode(prompt)
hf_inputs = transformers.BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, mm_kwargs)
return hf_inputs
def _unpad_audios(
self,
hf_inputs: "BatchFeature",
@@ -450,19 +428,6 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
deriving them from updates."""
return []
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask which modality owns each token of the expanded prompt, which is the
only marking of the tokens an expansion adds around an item."""
if any(mm_data.values()):
mm_data = {**mm_data, "return_mm_token_type_ids": True}
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_token_ids(self, modality: str) -> list[int]:
"""Token ids marking where `modality` sits in the prompt, which for some
processors differ from the placeholder written into it.
@@ -612,24 +577,55 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
with timing_ctx.record("apply_hf_processor"):
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
prompt = hf_processor.decode(prompt)
prompt_text = hf_processor.decode(prompt)
# Bypass cached processor and always apply to the full set of mm inputs
# NOTE: we can't just set caching=False because base class method
# transforms outputs to `MultiModalKwargs` which is not going to
# work for Transformers. The vision path has logic tied to
# `mm_tokens_per_modality` in _apply_vision()
prompt_ids, processed_data = self._apply_hf_processor_text_mm(
prompt_text=prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={
**hf_processor_mm_kwargs,
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
# (`_call_hf_processor` disables `add_special_tokens`).
"add_special_tokens": False,
},
hf_processor_mm_kwargs = {
**hf_processor_mm_kwargs,
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
# (the HF processor call disables `add_special_tokens`).
"add_special_tokens": False,
}
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
# Ask which modality owns each token of the expanded prompt, which
# is the only marking of the tokens an expansion adds around an item
if any(processor_data.values()):
processor_data = {**processor_data, "return_mm_token_type_ids": True}
try:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **processor_data),
hf_processor_mm_kwargs,
)
except ValueError:
if any(processor_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
prompt_ids = self.info.get_tokenizer().encode(prompt_text)
processed_data = transformers.BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
self._unpad_images(processed_data)
self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs)
processed_data.update(passthrough_data)
input_ids = processed_data.pop("input_ids")
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
# Use overrides if provided; fallback to data-dependent hashing.
with timing_ctx.record("get_mm_hashes"):
@@ -773,17 +769,41 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
except (AttributeError, KeyError, TypeError):
return None
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask for the replacement each placeholder expands to, and record it as
per-item fields: its token ids, and the tokens or patches behind them."""
mm_counts = mm_items.get_all_counts()
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
# Ask for the replacement each placeholder expands to, and record it as
# per-item fields: its token ids, and the tokens or patches behind them
if has_mm_data := any(mm_data.values()):
mm_data = {**mm_data, "return_text_replacement_offsets": True}
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
try:
hf_inputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
except ValueError:
if any(mm_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
tokenizer = self.info.get_tokenizer()
hf_inputs = transformers.BatchFeature(
dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)
# Drop the inputs the model would reject
hf_inputs.pop("mm_token_type_ids", None)
hf_inputs.pop("token_type_ids", None)
@@ -802,6 +822,8 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"so it can be fixed, and install `transformers<5.15.0` in the "
"meantime to locate placeholders in the expanded prompt instead."
)
hf_inputs.update(passthrough_data)
hf_inputs.pop("input_ids", None)
return hf_inputs
tokenizer = self.info.get_tokenizer()
@@ -829,6 +851,9 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
hf_inputs.update(passthrough_data)
hf_inputs.pop("input_ids")
return hf_inputs
+68 -25
View File
@@ -190,41 +190,84 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Text-only input not supported in composite processor
if not mm_data.get("audios", []):
prompt_ids = self.info.get_tokenizer().encode(
prompt, add_special_tokens=False
)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
audios = mm_data.get("audios", [])
assert isinstance(audios, list)
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_kwargs = dict(
**mm_kwargs,
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
include_audio_num_chunks=True,
)
item_processor_data = dict(**mm_data, audios=audios)
return mm_data, hf_processor_mm_kwargs
output = super()._call_hf_processor(
prompt=prompt,
mm_data=item_processor_data,
mm_kwargs=mm_kwargs,
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
output["audio_features"] = output.pop("audio_values")
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
return output
if processor_data:
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
processor_data,
hf_processor_mm_kwargs,
)
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
if prompt_text is not None:
processor_data = dict(text=prompt_text, **processor_data)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
def patched_call(**kwargs) -> BatchFeature:
# The remote UltravoxProcessor does not support the
# `truncation` kwarg injected by `call_hf_processor`: it
# passes `truncation` to its audio processor by itself while
# also forwarding `**kwargs` there, causing a duplicate
# keyword argument error. Its hardcoded `truncation=False`
# already matches vLLM's intended behavior.
kwargs.pop("truncation", None)
return hf_processor(**kwargs)
processed_data = self.info.ctx.call_hf_processor(
patched_call,
processor_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
else:
processed_data = BatchFeature(dict(passthrough_data))
return self._postprocess_hf_mm_data(
processor_data,
hf_processor_mm_kwargs,
processed_data,
)
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "audio_values" in processed_data:
processed_data["audio_features"] = processed_data.pop("audio_values")
return processed_data
def _get_mm_fields_config(
self,
+1 -1
View File
@@ -225,7 +225,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
) -> MultiModalProcessingInfo:
# The processor logic differs for single-image (crop) vs multi-image
# (no crop) requests. The processing cache assumes per-item output is
# invariant of how many images are passed per prompt, so we only cache
+23 -25
View File
@@ -47,12 +47,10 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
MultiModalProcessingInfo,
PlaceholderFeaturesInfo,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
TimingContext,
)
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
@@ -204,6 +202,9 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
# The tokens are already inserted by the chat template
hf_processor_applies_updates = True
def _get_mm_fields_config(
self,
hf_inputs: Mapping[str, NestedTensors],
@@ -220,12 +221,14 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
# skip validation here
pass
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -233,18 +236,23 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
# MistralCommonVoxtralProcessor accepts "audio"
mm_data["audio"] = audios
outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
# Avoid padding issue
dict(**mm_kwargs, return_tensors=None),
)
# Avoid padding issue
hf_processor_mm_kwargs = dict(**hf_processor_mm_kwargs, return_tensors=None)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
# Missing batch dimension
if is_list_of(outputs["input_ids"], int):
outputs["input_ids"] = [outputs["input_ids"]]
input_ids = processed_data.get("input_ids")
if is_list_of(input_ids, int):
processed_data["input_ids"] = [input_ids]
return outputs
return processed_data
def _get_prompt_updates(
self,
@@ -287,16 +295,6 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
),
]
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx)
# NOTE: The tokens are already inserted by the chat template
return prompt_ids, mm_info, True
@MULTIMODAL_REGISTRY.register_processor(
VoxtralMultiModalProcessor,
+27 -18
View File
@@ -756,27 +756,36 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
# for encoder profiling.
return [0]
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_data = dict(audio=mm_data.pop("audios"))
mm_kwargs = dict(
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
mm_data = dict(mm_data)
mm_data["audio"] = mm_data.pop("audios")
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
return processed_outputs
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "labels" in processed_data:
processed_data["input_ids"] = processed_data.pop("labels")
return processed_data
def _get_mm_fields_config(
self,