[MM] Simplify _apply_hf_processor_main (#53275)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -377,35 +377,41 @@ class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):
|
||||
class AudioFlamingo3MultiModalProcessor(
|
||||
BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, Any],
|
||||
) -> BatchFeature:
|
||||
processor_mm_data = dict(mm_data)
|
||||
audios = processor_mm_data.pop("audios", None)
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", None)
|
||||
if audios is not None:
|
||||
processor_mm_data["audio"] = audios
|
||||
mm_data["audio"] = audios
|
||||
|
||||
outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=processor_mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
if "input_features_mask" in outputs:
|
||||
outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "input_features_mask" in processed_data:
|
||||
processed_data["feature_attention_mask"] = processed_data.pop(
|
||||
"input_features_mask"
|
||||
)
|
||||
|
||||
audio_data = processor_mm_data.get("audio")
|
||||
audio_data = mm_data.get("audio")
|
||||
if audio_data is None:
|
||||
return outputs
|
||||
return processed_data
|
||||
|
||||
audio_list = audio_data if isinstance(audio_data, list) else [audio_data]
|
||||
if len(audio_list) == 0:
|
||||
return outputs
|
||||
return processed_data
|
||||
|
||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
feature_extractor = processor.feature_extractor
|
||||
sampling_rate = feature_extractor.sampling_rate
|
||||
chunk_length = feature_extractor.chunk_length
|
||||
@@ -423,8 +429,9 @@ class AudioFlamingo3MultiModalProcessor(
|
||||
n_win = max_windows
|
||||
chunk_counts.append(n_win)
|
||||
|
||||
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
|
||||
return outputs
|
||||
processed_data["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -467,24 +467,6 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
|
||||
|
||||
|
||||
class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
# HF processor always adds placeholders even when there's no image
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt_ids = tokenizer.encode(prompt)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
|
||||
return super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -135,34 +135,16 @@ class ChameleonDummyInputsBuilder(BaseDummyInputsBuilder[ChameleonProcessingInfo
|
||||
|
||||
|
||||
class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
# ChameleonProcessor requires text corresponding to the images
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
return super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
|
||||
def _apply_hf_processor_tokens_only(
|
||||
self,
|
||||
prompt_tokens: list[int],
|
||||
) -> list[int]:
|
||||
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||
# HF processor adds sep token for chat mode
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
sep_token_id = tokenizer.get_vocab()[tokenizer.sep_token] # type: ignore
|
||||
|
||||
sep_token_id = vocab[tokenizer.sep_token] # type: ignore
|
||||
|
||||
return prompt_tokens + [sep_token_id]
|
||||
return [*prompt, sep_token_id]
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -16,7 +16,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
from transformers import BatchFeature
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
@@ -492,14 +491,6 @@ class CheersDummyInputsBuilder(BaseDummyInputsBuilder[CheersProcessingInfo]):
|
||||
class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):
|
||||
"""Multimodal processor for Cheers model."""
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -222,24 +222,23 @@ class Cohere2VisionDummyInputsBuilder(
|
||||
class Cohere2VisionMultiModalProcessor(
|
||||
BaseMultiModalProcessor[Cohere2VisionProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt,
|
||||
mm_data,
|
||||
mm_kwargs,
|
||||
)
|
||||
if not mm_data:
|
||||
return processed_data
|
||||
|
||||
# Ensure num_patches is available for proper tensor splitting
|
||||
if (
|
||||
"num_patches" not in processed_outputs
|
||||
"num_patches" not in processed_data
|
||||
and (images := mm_data.get("images")) is not None
|
||||
):
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
# Fallback calculation if HF processor didn't provide num_patches
|
||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||
@@ -250,13 +249,13 @@ class Cohere2VisionMultiModalProcessor(
|
||||
image_width=parsed_images.get_image_size(i).width,
|
||||
image_height=parsed_images.get_image_size(i).height,
|
||||
processor=hf_processor,
|
||||
mm_kwargs=mm_kwargs,
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
for i in range(len(parsed_images))
|
||||
]
|
||||
processed_outputs["num_patches"] = torch.tensor(num_patches)
|
||||
processed_data["num_patches"] = torch.tensor(num_patches)
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import Any, ClassVar
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import BatchFeature, PretrainedConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
@@ -45,9 +45,12 @@ from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseProcessingInfo,
|
||||
EncDecMultiModalProcessor,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
TimingContext,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.processors.cohere_asr import (
|
||||
@@ -1941,27 +1944,51 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
|
||||
) -> list[int]:
|
||||
return [0]
|
||||
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
):
|
||||
if mm_data:
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
mm_data = dict(audio=dict(mm_data).pop("audios"))
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
if "labels" in processed_outputs:
|
||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
||||
return processed_outputs
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "labels" in processed_data:
|
||||
processed_data["input_ids"] = processed_data.pop("labels")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
# Dithering injects noise into the extracted features, so the
|
||||
# feature extractor is not a pure function of its input. Since the
|
||||
# processing cache assumes that processor outputs are invariant
|
||||
# across calls, bypass the cache when dithering is active.
|
||||
preproc = self.info.get_hf_config().preprocessor
|
||||
if preproc.get("dither", 1e-05) > 0:
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
return super()._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -154,16 +154,25 @@ class ColModernVBertDummyInputsBuilder(
|
||||
class ColModernVBertMultiModalProcessor(
|
||||
BaseMultiModalProcessor[ColModernVBertProcessingInfo],
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
text_encoding = tokenizer(
|
||||
prompt,
|
||||
prompt_text,
|
||||
return_tensors="pt",
|
||||
)
|
||||
result = BatchFeature(data=dict(text_encoding))
|
||||
@@ -183,7 +192,9 @@ class ColModernVBertMultiModalProcessor(
|
||||
)
|
||||
result.update(image_outputs)
|
||||
|
||||
return result
|
||||
processed_data = result
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
|
||||
from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
|
||||
from .interfaces import SupportsLateInteraction
|
||||
from .interfaces_base import default_pooling_type
|
||||
@@ -59,24 +60,31 @@ class ColPaliProcessingInfo(PaliGemmaProcessingInfo):
|
||||
class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
|
||||
"""Multimodal processor for ColPali."""
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
# The ColPali tokenizer_config.json ships with a small default
|
||||
# max_length (50) that truncates the 1024 image tokens inserted
|
||||
# by PaliGemmaProcessor, causing a token-count mismatch.
|
||||
# vLLM enforces its own max_model_len, so we disable HF
|
||||
# truncation to keep all image + text tokens intact.
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
dict(**mm_kwargs, truncation=False),
|
||||
)
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
# The ColPali tokenizer_config.json ships with a small default
|
||||
# max_length (50) that truncates the 1024 image tokens inserted
|
||||
# by PaliGemmaProcessor, causing a token-count mismatch.
|
||||
# vLLM enforces its own max_model_len, so we disable HF
|
||||
# truncation to keep all image + text tokens intact.
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
mm_data,
|
||||
dict(**hf_processor_mm_kwargs, truncation=False),
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
|
||||
@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
|
||||
|
||||
@@ -287,27 +287,34 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing
|
||||
class DeepseekOCRMultiModalProcessor(
|
||||
BaseMultiModalProcessor[DeepseekOCRProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
if mm_data:
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(prompt=prompt, **mm_data),
|
||||
mm_kwargs,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(prompt=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
else:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
processed_outputs = tokenizer(
|
||||
prompt, add_special_tokens=True, return_tensors="pt"
|
||||
processed_data = tokenizer(
|
||||
prompt_text, add_special_tokens=True, return_tensors="pt"
|
||||
)
|
||||
|
||||
return processed_outputs
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -158,27 +158,35 @@ class DeepseekOCR2DummyInputsBuilder(
|
||||
class DeepseekOCR2MultiModalProcessor(
|
||||
BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
if mm_data:
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(prompt=prompt, **mm_data),
|
||||
mm_kwargs,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(prompt=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
else:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
processed_outputs = tokenizer(
|
||||
prompt, add_special_tokens=True, return_tensors="pt"
|
||||
processed_data = tokenizer(
|
||||
prompt_text, add_special_tokens=True, return_tensors="pt"
|
||||
)
|
||||
|
||||
return processed_outputs
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -43,7 +43,6 @@ from vllm.multimodal.processing.processor import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.transformers_utils.configs.deepseek_vl2 import (
|
||||
DeepseekVLV2Config,
|
||||
MlpProjectorConfig,
|
||||
@@ -236,28 +235,23 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
|
||||
class DeepseekVL2MultiModalProcessor(
|
||||
BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
return tokenizer(prompt, add_special_tokens=True, return_tensors="pt")
|
||||
return processed_data
|
||||
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
processed_data["num_patches"] = (
|
||||
processed_data["images_spatial_crop"].prod(-1) + 1
|
||||
)
|
||||
|
||||
processed_outputs["num_patches"] = (
|
||||
processed_outputs["images_spatial_crop"].prod(-1) + 1
|
||||
)
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
@@ -315,7 +309,7 @@ class DeepseekVL2MultiModalProcessor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
||||
) -> MultiModalProcessingInfo:
|
||||
# The processor logic is different for len(images) <= 2 vs > 2
|
||||
# Since the processing cache assumes that the processor output is
|
||||
# invariant of how many images are passed per prompt, we only
|
||||
|
||||
@@ -1110,21 +1110,20 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
pixel_values = pixel_values.to(hf_config.dtype)
|
||||
return pixel_values
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# when the prompt is not empty but the multimodal data is empty,
|
||||
# directly invoke the tokenizer.
|
||||
if "images" not in mm_data and "videos" not in mm_data and prompt != "":
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt_ids = tokenizer.encode(prompt)
|
||||
tokenizer_output = BatchFeature(
|
||||
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
||||
)
|
||||
return tokenizer_output
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if "images" not in mm_data and "videos" not in mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
if "images" not in mm_data:
|
||||
mm_data["images"] = []
|
||||
@@ -1132,7 +1131,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
mm_data["videos"] = []
|
||||
|
||||
# Check if HF processor supports video metadata
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
supports_video_metadata = getattr(
|
||||
hf_processor, "supports_video_metadata", False
|
||||
)
|
||||
@@ -1149,8 +1148,10 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
|
||||
processor_output = self.info.ctx.call_hf_processor(
|
||||
hf_processor,
|
||||
dict(text=[prompt], images=mm_data["images"], videos=mm_data["videos"]),
|
||||
mm_kwargs,
|
||||
dict(
|
||||
text=[prompt_text], images=mm_data["images"], videos=mm_data["videos"]
|
||||
),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# Divide the processor_output into two modalities: image and video.
|
||||
@@ -1158,7 +1159,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
pixel_values = processor_output["images"]
|
||||
if pixel_values is not None:
|
||||
processor_output["images"] = self._pixel_values_norm(
|
||||
pixel_values, mm_kwargs
|
||||
pixel_values, hf_processor_mm_kwargs
|
||||
)
|
||||
for key in list(processor_output.keys()):
|
||||
if processor_output[key] is None:
|
||||
@@ -1184,7 +1185,9 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
]
|
||||
del processor_output["images"]
|
||||
|
||||
return processor_output
|
||||
processed_data = processor_output
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -228,27 +228,36 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing
|
||||
class FireRedASR2MultiModalProcessor(
|
||||
BaseMultiModalProcessor[FireRedASR2ProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
mm_data = dict(audio=mm_data.pop("audios"))
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
if "labels" in processed_outputs:
|
||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
||||
return processed_outputs
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "labels" in processed_data:
|
||||
processed_data["input_ids"] = processed_data.pop("labels")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -451,27 +451,36 @@ class FireRedLIDMultiModalProcessor(
|
||||
# Dummy encoder prompt for profiling (encoder only processes audio).
|
||||
return [0]
|
||||
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
mm_data = dict(audio=mm_data.pop("audios"))
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
if "labels" in processed_outputs:
|
||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
||||
return processed_outputs
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "labels" in processed_data:
|
||||
processed_data["input_ids"] = processed_data.pop("labels")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -44,9 +44,12 @@ from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
TimingContext,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -747,27 +750,50 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
|
||||
|
||||
|
||||
class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
mm_data = dict(audio=mm_data.pop("audios"))
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
if "labels" in processed_outputs:
|
||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
||||
return processed_outputs
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "labels" in processed_data:
|
||||
processed_data["input_ids"] = processed_data.pop("labels")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
# Dithering injects noise into the extracted features, so the
|
||||
# feature extractor is not a pure function of its input. Since the
|
||||
# processing cache assumes that processor outputs are invariant
|
||||
# across calls, bypass the cache when dithering is active.
|
||||
if self.info.get_feature_extractor().dither > 0:
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
return super()._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -612,20 +612,31 @@ class FunAudioChatDummyInputsBuilder(
|
||||
class FunAudioChatMultiModalProcessor(
|
||||
BaseMultiModalProcessor[FunAudioChatProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt)])
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
|
||||
|
||||
audios = mm_data.get("audios", [])
|
||||
if not audios:
|
||||
return BatchFeature({"input_ids": input_ids})
|
||||
processed_data = BatchFeature({"input_ids": input_ids})
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
sr = int(feature_extractor.sampling_rate)
|
||||
min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400)
|
||||
|
||||
@@ -676,7 +687,9 @@ class FunAudioChatMultiModalProcessor(
|
||||
"feature_exist_mask": torch.ones((len(wavs),), dtype=torch.bool),
|
||||
}
|
||||
|
||||
return BatchFeature({"input_ids": input_ids, **mm_inputs})
|
||||
processed_data = BatchFeature({"input_ids": input_ids, **mm_inputs})
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -261,39 +261,38 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
|
||||
|
||||
|
||||
class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt,
|
||||
mm_data,
|
||||
mm_kwargs,
|
||||
)
|
||||
if not mm_data:
|
||||
return processed_data
|
||||
|
||||
# HF processor pops the `num_crops` kwarg, which is needed by vLLM
|
||||
if (images := mm_data.get("images")) is not None:
|
||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = [
|
||||
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
||||
]
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
num_crops = [
|
||||
self.info.get_num_crops(
|
||||
image_width=size.width,
|
||||
image_height=size.height,
|
||||
processor=hf_processor,
|
||||
mm_kwargs=mm_kwargs,
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
for size in image_sizes
|
||||
]
|
||||
processed_outputs["num_patches"] = torch.tensor(num_crops) + 1
|
||||
processed_data["num_patches"] = torch.tensor(num_crops) + 1
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import inspect
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
@@ -257,40 +258,69 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
|
||||
|
||||
|
||||
class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# HF Transformers audio processor no longer accepts `audios` key.
|
||||
# We pop `audios` and replace it with `audio` key to suppress
|
||||
# the warning.
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
if "audios" in mm_data:
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt,
|
||||
mm_data,
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
if "input_features" in processed_outputs:
|
||||
# Padding enables audio_tower to run in batched mode
|
||||
processed_outputs["input_features_padded"] = processed_outputs[
|
||||
"input_features"
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "input_features" in processed_data:
|
||||
# The feature extractor pads every clip to the longest one in
|
||||
# the batch, so each clip's frame count depends on the other
|
||||
# clips it was processed with. Trim each clip to the number of
|
||||
# frames it would have if processed alone so that each item's
|
||||
# output (and thus its cache entry) is independent of the other
|
||||
# items in the batch. `batch_audio_features` re-pads the
|
||||
# per-item features so audio_tower can still run batched.
|
||||
num_frames = [
|
||||
self._get_num_audio_frames_alone(len(audio))
|
||||
for audio in mm_data["audio"]
|
||||
]
|
||||
processed_data["input_features_padded"] = [
|
||||
features[:n]
|
||||
for features, n in zip(processed_data.pop("input_features"), num_frames)
|
||||
]
|
||||
processed_data["input_features_mask"] = [
|
||||
mask[:n]
|
||||
for mask, n in zip(processed_data["input_features_mask"], num_frames)
|
||||
]
|
||||
|
||||
# Unpad features here since we need the output of each item to be
|
||||
# independent of other items for the cache to work correctly
|
||||
unpadded_features = [
|
||||
f[mask]
|
||||
for f, mask in zip(
|
||||
processed_outputs["input_features"],
|
||||
processed_outputs["input_features_mask"],
|
||||
)
|
||||
]
|
||||
processed_outputs["input_features"] = unpadded_features
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_num_audio_frames_alone(self, num_samples: int) -> int:
|
||||
"""
|
||||
Get the number of mel frames that the feature extractor produces
|
||||
for a single audio clip of the given length, independent of batch
|
||||
padding.
|
||||
"""
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
call_params = inspect.signature(type(feature_extractor).__call__).parameters
|
||||
max_length = call_params["max_length"].default
|
||||
pad_to_multiple_of = call_params["pad_to_multiple_of"].default
|
||||
|
||||
num_samples = min(num_samples, max_length)
|
||||
remainder = num_samples % pad_to_multiple_of
|
||||
if remainder:
|
||||
num_samples += pad_to_multiple_of - remainder
|
||||
|
||||
frame_size = feature_extractor.frame_length + 1
|
||||
num_frames = (num_samples - frame_size) // feature_extractor.hop_length + 1
|
||||
|
||||
return max(num_frames, 0)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -560,13 +560,22 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
|
||||
|
||||
|
||||
class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
|
||||
|
||||
if val is not None and val not in _SUPPORTED_SOFT_TOKENS:
|
||||
@@ -611,15 +620,15 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
timestamps = [idx / fps for idx in frame_indices]
|
||||
|
||||
# Process frames as images with max_soft_tokens=70
|
||||
video_mm_kwargs = dict(mm_kwargs)
|
||||
video_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||
video_mm_kwargs["max_soft_tokens"] = _VIDEO_MAX_SOFT_TOKENS
|
||||
|
||||
dummy_prompt = ("\t" + processor.image_token) * len(frames)
|
||||
|
||||
frame_outputs = super()._call_hf_processor(
|
||||
prompt=dummy_prompt,
|
||||
mm_data={"images": frames},
|
||||
mm_kwargs=video_mm_kwargs,
|
||||
frame_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**video_mm_kwargs),
|
||||
dict(text=dummy_prompt, **{"images": frames}),
|
||||
video_mm_kwargs,
|
||||
)
|
||||
|
||||
# Remap HF key name
|
||||
@@ -660,7 +669,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
# split-replace bug where replacement text (which itself
|
||||
# contains <|video|> tokens) collides with later splits.
|
||||
vt = processor.video_token
|
||||
parts = prompt.split(vt, len(video_replacements))
|
||||
parts = prompt_text.split(vt, len(video_replacements))
|
||||
|
||||
# NOTE: len(parts) <= len(video_replacements) + 1
|
||||
parts_with_repl: list[str] = []
|
||||
@@ -668,7 +677,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
parts_with_repl.extend([part, repl])
|
||||
parts_with_repl.extend(parts[len(video_replacements) :])
|
||||
|
||||
prompt = "".join(parts_with_repl)
|
||||
prompt_text = "".join(parts_with_repl)
|
||||
|
||||
video_outputs = {
|
||||
"pixel_values_videos": torch.cat(all_video_pixel_values, dim=0),
|
||||
@@ -712,45 +721,46 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
# NOTE: This requires a corresponding type annotation on the
|
||||
# HF side (Gemma4ProcessorKwargs.images_kwargs) so that
|
||||
# _merge_kwargs routes max_soft_tokens into images_kwargs.
|
||||
patched_mm_kwargs = dict(mm_kwargs)
|
||||
patched_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||
if val is not None and is_top_level_max_soft_tokens:
|
||||
patched_mm_kwargs["max_soft_tokens"] = val
|
||||
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt,
|
||||
mm_data,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**patched_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
patched_mm_kwargs,
|
||||
)
|
||||
|
||||
# HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'.
|
||||
# Remap here to keep a single translation point.
|
||||
if "image_position_ids" in processed_outputs:
|
||||
processed_outputs["pixel_position_ids"] = processed_outputs.pop(
|
||||
if "image_position_ids" in processed_data:
|
||||
processed_data["pixel_position_ids"] = processed_data.pop(
|
||||
"image_position_ids"
|
||||
)
|
||||
|
||||
if "input_features" in processed_outputs:
|
||||
if "input_features" in processed_data:
|
||||
# Unpad per-item so each item's cache entry is
|
||||
# self-contained. The batched() field config in
|
||||
# _get_mm_fields_config stacks equal-length items and leaves
|
||||
# the rest as a list; _process_audio_input re-pads that list
|
||||
# to the batch's max length.
|
||||
masks = processed_outputs["input_features_mask"]
|
||||
masks = processed_data["input_features_mask"]
|
||||
unpadded_features = [
|
||||
f[mask]
|
||||
for f, mask in zip(
|
||||
processed_outputs["input_features"],
|
||||
processed_data["input_features"],
|
||||
masks,
|
||||
)
|
||||
]
|
||||
unpadded_masks = [mask[mask] for mask in masks]
|
||||
processed_outputs["input_features"] = unpadded_features
|
||||
processed_outputs["input_features_padded"] = unpadded_features
|
||||
processed_outputs["input_features_mask"] = unpadded_masks
|
||||
processed_data["input_features"] = unpadded_features
|
||||
processed_data["input_features_padded"] = unpadded_features
|
||||
processed_data["input_features_mask"] = unpadded_masks
|
||||
|
||||
# Merge video outputs into the final result
|
||||
combined_outputs = dict(processed_outputs, **video_outputs)
|
||||
return BatchFeature(combined_outputs)
|
||||
processed_data.update(video_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
@@ -815,7 +825,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
image_size = images.get_image_size(item_idx)
|
||||
# Resolve the effective max_soft_tokens by merging
|
||||
# per-prompt kwargs with the config-level defaults,
|
||||
# consistent with how _call_hf_processor resolves it.
|
||||
# consistent with how _apply_hf_processor_main resolves it.
|
||||
# Without this merge, a missing per-prompt override
|
||||
# would fall back to vision_cfg.default_output_length
|
||||
# instead of the config's mm_processor_kwargs default.
|
||||
|
||||
@@ -1576,32 +1576,39 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
|
||||
return prepared_data, prepared_kwargs
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
if not mm_data:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
use_direct_path = (
|
||||
not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
|
||||
)
|
||||
if use_direct_path:
|
||||
prepared_data, prepared_kwargs = self._get_direct_path_inputs(
|
||||
mm_data, mm_kwargs
|
||||
mm_data, hf_processor_mm_kwargs
|
||||
)
|
||||
return super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=prepared_data,
|
||||
mm_kwargs=prepared_kwargs,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**prepared_kwargs),
|
||||
dict(text=prompt_text, **prepared_data),
|
||||
prepared_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
if (
|
||||
"videos" in mm_data
|
||||
@@ -1616,7 +1623,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
video_array, metadata = item
|
||||
|
||||
# don't update mm_kwargs inplace
|
||||
video_mm_kwargs = dict(**mm_kwargs)
|
||||
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
|
||||
video_mm_kwargs["do_sample_frames"] = metadata.get(
|
||||
"do_sample_frames", True
|
||||
)
|
||||
@@ -1625,10 +1632,13 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
video_mm_data["videos"] = [[video_array]]
|
||||
video_mm_data["video_metadata"] = [[_to_video_metadata(metadata)]]
|
||||
|
||||
video_outputs = super()._call_hf_processor(
|
||||
prompt="<|begin_of_video|><|video|><|end_of_video|>",
|
||||
mm_data=video_mm_data,
|
||||
mm_kwargs=video_mm_kwargs,
|
||||
video_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**video_mm_kwargs),
|
||||
dict(
|
||||
text="<|begin_of_video|><|video|><|end_of_video|>",
|
||||
**video_mm_data,
|
||||
),
|
||||
video_mm_kwargs,
|
||||
)
|
||||
input_ids = video_outputs.pop("input_ids")
|
||||
if swap_video_frame_tokens:
|
||||
@@ -1636,7 +1646,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
processor.video_token_id
|
||||
)
|
||||
video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
|
||||
prompt = prompt.replace(
|
||||
prompt_text = prompt_text.replace(
|
||||
"<|begin_of_video|><|video|><|end_of_video|>",
|
||||
video_placeholder,
|
||||
1,
|
||||
@@ -1652,20 +1662,20 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
video_outputs = dict()
|
||||
swap_video_frame_tokens = False
|
||||
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
if swap_video_frame_tokens:
|
||||
input_ids = processed_outputs["input_ids"]
|
||||
input_ids = processed_data["input_ids"]
|
||||
input_ids[input_ids == processor.video_token_id] = processor.image_token_id
|
||||
processed_outputs["input_ids"] = input_ids
|
||||
combined_outputs = dict(
|
||||
processed_outputs,
|
||||
**video_outputs,
|
||||
)
|
||||
return BatchFeature(combined_outputs)
|
||||
processed_data["input_ids"] = input_ids
|
||||
|
||||
processed_data.update(video_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -750,46 +750,40 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
|
||||
chunk_counts.append(min(n_chunks, max_windows))
|
||||
return chunk_counts
|
||||
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: dict[str, object],
|
||||
mm_kwargs: Mapping[str, Any],
|
||||
) -> BatchFeature:
|
||||
# Normalize input: handle deprecated key and list conversion.
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
if "audios" in mm_data:
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
audio = mm_data.get("audio", [])
|
||||
audio_list = [audio] if audio and not isinstance(audio, list) else audio
|
||||
|
||||
# Early return for text-only.
|
||||
if not audio_list:
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
|
||||
# Handle sampling_rate
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
|
||||
# Call parent method
|
||||
outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
# Postprocess: rename mask and add chunk counts
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
# Handle different key names from different transformers versions
|
||||
if "input_features_mask" in outputs:
|
||||
outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
|
||||
elif "input_features_mask" not in outputs and "input_features" in outputs:
|
||||
if "input_features_mask" in processed_data:
|
||||
processed_data["feature_attention_mask"] = processed_data.pop(
|
||||
"input_features_mask"
|
||||
)
|
||||
elif "input_features" in processed_data:
|
||||
# If no mask is provided, create one from input_features
|
||||
input_features = outputs["input_features"]
|
||||
input_features = processed_data["input_features"]
|
||||
if isinstance(input_features, torch.Tensor):
|
||||
# Create a mask of all ones matching the sequence length
|
||||
mask = torch.ones(
|
||||
@@ -797,18 +791,22 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
|
||||
input_features.shape[-1],
|
||||
dtype=torch.long,
|
||||
)
|
||||
outputs["feature_attention_mask"] = mask
|
||||
processed_data["feature_attention_mask"] = mask
|
||||
|
||||
# Get processor for chunk counts calculation
|
||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
audio = mm_data.get("audio", [])
|
||||
audio_list = [audio] if audio and not isinstance(audio, list) else audio
|
||||
if audio_list:
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
# Override chunk counts calculation with GLM-ASR specific logic
|
||||
chunk_counts = self._calculate_chunk_counts(
|
||||
audio_list, processor.feature_extractor, processor
|
||||
)
|
||||
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
|
||||
# Override chunk counts calculation with GLM-ASR specific logic
|
||||
chunk_counts = self._calculate_chunk_counts(
|
||||
audio_list, processor.feature_extractor, processor
|
||||
)
|
||||
processed_data["chunk_counts"] = torch.tensor(
|
||||
chunk_counts, dtype=torch.long
|
||||
)
|
||||
|
||||
return outputs
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -180,12 +180,14 @@ class GraniteSpeechMultiModalProcessor(
|
||||
)
|
||||
]
|
||||
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
|
||||
@@ -193,21 +195,23 @@ class GraniteSpeechMultiModalProcessor(
|
||||
# GraniteSpeechFeatureExtractor accepts "audio"
|
||||
mm_data["audio"] = audios
|
||||
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "audio" in mm_data:
|
||||
# Calculate the number of audio tokens per entry in the batch;
|
||||
# This is used to split the batch back out after padding.
|
||||
audio_token_index = self.info.get_hf_config().audio_token_index
|
||||
processed_outputs["audio_embed_sizes"] = (
|
||||
processed_outputs["input_ids"] == audio_token_index
|
||||
processed_data["audio_embed_sizes"] = (
|
||||
processed_data["input_ids"] == audio_token_index
|
||||
).sum(-1)
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
|
||||
class GraniteSpeechDummyInputsBuilder(
|
||||
|
||||
@@ -135,7 +135,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
||||
) -> MultiModalProcessingInfo:
|
||||
# The processor logic is different for len(images) <= 1 vs > 1
|
||||
# Since the processing cache assumes that the processor output is
|
||||
# invariant of how many images are passed per prompt, we only
|
||||
|
||||
@@ -731,28 +731,6 @@ class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo
|
||||
|
||||
|
||||
class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
# HunYuanVLProcessor requires image placeholders wrapped with start/end tokens.
|
||||
if mm_data.get("images") is not None and prompt:
|
||||
img_tok = hf_processor.image_token
|
||||
wrapped = (
|
||||
f"{hf_processor.image_start_token}{img_tok}"
|
||||
f"{hf_processor.image_end_token}"
|
||||
)
|
||||
if img_tok in prompt and wrapped not in prompt:
|
||||
prompt = prompt.replace(img_tok, wrapped)
|
||||
return self.info.ctx.call_hf_processor(
|
||||
hf_processor,
|
||||
dict(text=prompt, **mm_data),
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -190,77 +190,71 @@ class HCXVisionDummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionProcessingInfo
|
||||
|
||||
|
||||
class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
for video_idx, video_arr in enumerate(mm_data.get("videos", [])):
|
||||
if video_arr.dtype != np.uint8:
|
||||
mm_data["videos"][video_idx] = video_arr.astype(np.uint8)
|
||||
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
hf_processor=self.info.get_hf_processor(**mm_kwargs),
|
||||
images = mm_data.get("images")
|
||||
videos = mm_data.get("videos")
|
||||
|
||||
# batchify input as a single item
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
hf_processor=self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
data=dict(
|
||||
text=prompt,
|
||||
images=None,
|
||||
videos=None,
|
||||
images=None if images is None else [images],
|
||||
videos=None if videos is None else [videos],
|
||||
),
|
||||
) # text-only
|
||||
)
|
||||
|
||||
if len(mm_data) > 0:
|
||||
images = mm_data.get("images")
|
||||
videos = mm_data.get("videos")
|
||||
for k, v in processed_data.items():
|
||||
if isinstance(v, list) and len(v) > 0:
|
||||
assert len(v) == 1
|
||||
processed_data[k] = v[0]
|
||||
|
||||
# batchify input as a single item
|
||||
_processed_outputs = self.info.ctx.call_hf_processor(
|
||||
hf_processor=self.info.get_hf_processor(**mm_kwargs),
|
||||
data=dict(
|
||||
text=None,
|
||||
images=None if images is None else [images],
|
||||
videos=None if videos is None else [videos],
|
||||
),
|
||||
) # mm-only
|
||||
if images:
|
||||
processed_data["image_sizes_images"] = torch.tensor(
|
||||
processed_data["image_sizes_images"]
|
||||
)
|
||||
processed_data["vision_query_lengths_images"] = torch.tensor(
|
||||
processed_data["vision_query_lengths_images"]
|
||||
)
|
||||
|
||||
for k, v in _processed_outputs.items():
|
||||
if isinstance(v, list) and len(v) > 0:
|
||||
assert len(v) == 1
|
||||
_processed_outputs[k] = v[0]
|
||||
|
||||
if images:
|
||||
_processed_outputs["image_sizes_images"] = torch.tensor(
|
||||
_processed_outputs["image_sizes_images"]
|
||||
)
|
||||
_processed_outputs["vision_query_lengths_images"] = torch.tensor(
|
||||
_processed_outputs["vision_query_lengths_images"]
|
||||
)
|
||||
|
||||
if videos:
|
||||
_idx_per_video = [
|
||||
0,
|
||||
*accumulate(
|
||||
get_num_combined_frames(len(video)) for video in videos
|
||||
),
|
||||
if videos:
|
||||
_idx_per_video = [
|
||||
0,
|
||||
*accumulate(get_num_combined_frames(len(video)) for video in videos),
|
||||
]
|
||||
processed_data["pixel_values_videos"] = [
|
||||
processed_data["pixel_values_videos"][
|
||||
_idx_per_video[i] : _idx_per_video[i + 1]
|
||||
]
|
||||
_processed_outputs["pixel_values_videos"] = [
|
||||
_processed_outputs["pixel_values_videos"][
|
||||
for i in range(len(videos))
|
||||
]
|
||||
processed_data["vision_query_lengths_videos"] = [
|
||||
torch.tensor(
|
||||
processed_data["vision_query_lengths_videos"][
|
||||
_idx_per_video[i] : _idx_per_video[i + 1]
|
||||
]
|
||||
for i in range(len(videos))
|
||||
]
|
||||
_processed_outputs["vision_query_lengths_videos"] = [
|
||||
torch.tensor(
|
||||
_processed_outputs["vision_query_lengths_videos"][
|
||||
_idx_per_video[i] : _idx_per_video[i + 1]
|
||||
]
|
||||
)
|
||||
for i in range(len(videos))
|
||||
]
|
||||
)
|
||||
for i in range(len(videos))
|
||||
]
|
||||
|
||||
processed_outputs.update(_processed_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -261,34 +261,44 @@ class HCXVisionV2MultiModalProcessor(
|
||||
):
|
||||
"""Multimodal processor for HyperCLOVAX V2 (32B Think model)."""
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
images = mm_data.get("images")
|
||||
videos = mm_data.get("videos")
|
||||
|
||||
# Get the HF processor
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
# Build data dict for HF processor (images/videos only)
|
||||
# NOTE: We pass the prompt as-is without token normalization.
|
||||
# Token expansion is handled by vLLM via _get_prompt_updates.
|
||||
data: dict[str, object] = dict(
|
||||
text=prompt,
|
||||
text=prompt_text,
|
||||
images=images,
|
||||
videos=videos,
|
||||
)
|
||||
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
hf_processor=hf_processor,
|
||||
data=data,
|
||||
kwargs=mm_kwargs,
|
||||
kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -239,51 +239,58 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
|
||||
|
||||
|
||||
class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Text-only input not supported in composite processor
|
||||
if not (images := mm_data.get("images", [])):
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
image_processor = self.info.get_hf_processor().image_processor
|
||||
if getattr(image_processor, "backend", "pil") == "pil":
|
||||
mm_kwargs = {"input_data_format": "channels_last", **mm_kwargs}
|
||||
hf_processor_mm_kwargs = {
|
||||
"input_data_format": "channels_last",
|
||||
**hf_processor_mm_kwargs,
|
||||
}
|
||||
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
mm_data,
|
||||
mm_kwargs,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
images = mm_data.get("images", [])
|
||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = [
|
||||
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
||||
]
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
num_patches = [
|
||||
self.info.get_num_patches(
|
||||
image_width=size.width,
|
||||
image_height=size.height,
|
||||
processor=hf_processor,
|
||||
mm_kwargs=mm_kwargs,
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
for size in image_sizes
|
||||
]
|
||||
processed_outputs["num_patches"] = torch.tensor(num_patches)
|
||||
processed_data["num_patches"] = torch.tensor(num_patches)
|
||||
|
||||
# Remove the extra batch dimension
|
||||
processed_outputs["pixel_values"].squeeze_(0)
|
||||
processed_outputs["pixel_attention_mask"].squeeze_(0)
|
||||
processed_data["pixel_values"].squeeze_(0)
|
||||
processed_data["pixel_attention_mask"].squeeze_(0)
|
||||
|
||||
return processed_outputs
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -332,19 +332,28 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
|
||||
class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]):
|
||||
"""Basic image-only MultiModalProcessor for InternS1-style models."""
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
videos = mm_data.pop("videos", [])
|
||||
images = mm_data.pop("images", [])
|
||||
assert isinstance(videos, list)
|
||||
assert isinstance(images, list)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = hf_processor.tokenizer
|
||||
video_token_id = tokenizer.encode(
|
||||
hf_processor.video_token, add_special_tokens=False
|
||||
@@ -352,23 +361,29 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
assert len(video_token_id) == 1
|
||||
video_token_id = video_token_id[0]
|
||||
|
||||
prompt = re.sub(hf_processor.image_token, "<image_placeholder>", prompt)
|
||||
prompt = re.sub(hf_processor.video_token, "<video_placeholder>", prompt)
|
||||
prompt_text = re.sub(
|
||||
hf_processor.image_token, "<image_placeholder>", prompt_text
|
||||
)
|
||||
prompt_text = re.sub(
|
||||
hf_processor.video_token, "<video_placeholder>", prompt_text
|
||||
)
|
||||
|
||||
image_outputs = {}
|
||||
if images:
|
||||
image_pixel_values = []
|
||||
for image in images:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=hf_processor.image_token,
|
||||
mm_data={"images": image},
|
||||
mm_kwargs=mm_kwargs,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=hf_processor.image_token, **{"images": image}),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
image_pixel_values.append(processed_outputs.pop("pixel_values"))
|
||||
image_pixel_values.append(processed_data.pop("pixel_values"))
|
||||
|
||||
input_ids = processed_outputs.pop("input_ids")
|
||||
input_ids = processed_data.pop("input_ids")
|
||||
image_placeholder = tokenizer.batch_decode(input_ids)[0]
|
||||
prompt = prompt.replace("<image_placeholder>", image_placeholder, 1)
|
||||
prompt_text = prompt_text.replace(
|
||||
"<image_placeholder>", image_placeholder, 1
|
||||
)
|
||||
|
||||
num_patches = [len(item) for item in image_pixel_values]
|
||||
image_outputs = {
|
||||
@@ -381,18 +396,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
if videos:
|
||||
video_pixel_values = []
|
||||
for video in videos:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=hf_processor.video_token,
|
||||
mm_data={"videos": video},
|
||||
mm_kwargs=mm_kwargs,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=hf_processor.video_token, **{"videos": video}),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
video_pixel_values.append(processed_outputs.pop("pixel_values"))
|
||||
video_pixel_values.append(processed_data.pop("pixel_values"))
|
||||
|
||||
input_ids = processed_outputs.pop("input_ids")
|
||||
input_ids = processed_data.pop("input_ids")
|
||||
input_ids[input_ids == hf_processor.image_token_id] = video_token_id
|
||||
|
||||
video_placeholder = tokenizer.batch_decode(input_ids)[0]
|
||||
prompt = prompt.replace("<video_placeholder>", video_placeholder, 1)
|
||||
prompt_text = prompt_text.replace(
|
||||
"<video_placeholder>", video_placeholder, 1
|
||||
)
|
||||
|
||||
num_frames = [len(item) for item in video_pixel_values]
|
||||
video_outputs = {
|
||||
@@ -401,11 +418,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
"video_token_id": torch.tensor(video_token_id),
|
||||
}
|
||||
|
||||
prompt = re.sub("<image_placeholder>", hf_processor.image_token, prompt)
|
||||
prompt = re.sub("<video_placeholder>", hf_processor.video_token, prompt)
|
||||
text_outputs = tokenizer(prompt, return_tensors="pt")
|
||||
prompt_text = re.sub(
|
||||
"<image_placeholder>", hf_processor.image_token, prompt_text
|
||||
)
|
||||
prompt_text = re.sub(
|
||||
"<video_placeholder>", hf_processor.video_token, prompt_text
|
||||
)
|
||||
text_outputs = tokenizer(prompt_text, return_tensors="pt")
|
||||
|
||||
return BatchFeature({**text_outputs, **image_outputs, **video_outputs})
|
||||
processed_data.update(text_outputs)
|
||||
processed_data.update(image_outputs)
|
||||
processed_data.update(video_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -220,27 +220,21 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
|
||||
class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
"""Basic image-only MultiModalProcessor for InternVL-style models."""
|
||||
|
||||
def _call_hf_processor(
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token_id = hf_processor.ctx_image_token_id
|
||||
|
||||
# Since there may be extra tokens in the feature placeholders,
|
||||
# we need to pass the image token ID to the model to select the
|
||||
# tokens to merge from the vision encoder outputs
|
||||
processed_outputs["image_token_id"] = torch.tensor(image_token_id)
|
||||
processed_data["image_token_id"] = torch.tensor(image_token_id)
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_image_fields_config(self, hf_inputs: BatchFeature):
|
||||
image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
|
||||
@@ -455,19 +449,23 @@ class InternVLMultiModalProcessor(
|
||||
):
|
||||
"""InternVL MultiModalProcessor extended for video support"""
|
||||
|
||||
def _call_hf_processor(
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
processed_data = super()._postprocess_hf_mm_data(
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
processed_data,
|
||||
)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
if (video_token_id := hf_processor.ctx_video_token_id) is not None:
|
||||
processed_outputs["video_token_id"] = torch.tensor(video_token_id)
|
||||
processed_data["video_token_id"] = torch.tensor(video_token_id)
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_video_fields_config(self, hf_inputs: BatchFeature):
|
||||
video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.pooler import DispatchPooler
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsCrossEncoding, SupportsMultiModal, SupportsScoreTemplate
|
||||
@@ -54,19 +55,28 @@ class JinaVLScorer(nn.Module):
|
||||
|
||||
|
||||
class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# NOTE: We should reverse the order of the mm_data because the
|
||||
# query prompt is placed after the document prompt in the score
|
||||
# template for JinaVLForRanking model, but in mm_data they are
|
||||
# stored in the opposite order (query first, then document).
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
for _, value in mm_data.items():
|
||||
value.reverse()
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
|
||||
@@ -461,17 +461,21 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
||||
def media_token_id(self) -> int:
|
||||
return self.info.get_hf_config().text_config.eos_token_id + 1
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Run the underlying HF processor on text and image data."""
|
||||
# Text-only input is handled as a special case here.
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data or not mm_data.get("images", []):
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
# Images
|
||||
image_inputs = mm_data.get("images", [])
|
||||
@@ -493,7 +497,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
media_token = tokenizer.convert_ids_to_tokens([self.media_token_id])[0]
|
||||
prompt_replaced = prompt.replace("<image>", media_token)
|
||||
prompt_replaced = prompt_text.replace("<image>", media_token)
|
||||
input_ids = tokenizer.encode(prompt_replaced)
|
||||
input_ids = torch.tensor(input_ids)
|
||||
|
||||
@@ -527,7 +531,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
||||
image_token_thw=torch.tensor(image_meta["image_token_thw"]),
|
||||
pixel_sizes=torch.tensor(pixel_sizes),
|
||||
)
|
||||
return BatchFeature(combined_outputs, tensor_type="pt")
|
||||
processed_data = BatchFeature(combined_outputs, tensor_type="pt")
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -1162,15 +1162,8 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):
|
||||
|
||||
|
||||
class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override to use the text path instead of token path to use the
|
||||
# video-specific logic in processing_keye.py
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -378,15 +378,8 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
|
||||
|
||||
|
||||
class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override to use the text path instead of token path to use the
|
||||
# video-specific logic in processing_keye.py
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -37,6 +37,7 @@ from vllm.multimodal.parse import (
|
||||
DictEmbeddingItems,
|
||||
ModalityData,
|
||||
ModalityDataItems,
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
@@ -226,13 +227,20 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser):
|
||||
class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]):
|
||||
"""vLLM multi-modal processor wrapper for Kimi-Audio."""
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Call the HuggingFace processor."""
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
# Convert mm_data format: {'audios': [...]} -> {'audio': ...}
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
@@ -252,11 +260,13 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
|
||||
mm_data["audio"] = audio_arrays
|
||||
|
||||
# Use the context's call_hf_processor for proper handling
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
mm_kwargs,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -258,16 +258,6 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
|
||||
grid_thws=MultiModalFieldConfig.batched("vision_chunk", keep_on_cpu=True),
|
||||
)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override to use the text path instead of token path because vision chunk
|
||||
# is not considered
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -237,6 +237,9 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
|
||||
|
||||
|
||||
class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -393,45 +393,38 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):
|
||||
|
||||
|
||||
class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
# Text-only input not supported in composite processor
|
||||
if not (images := mm_data.get("images", [])):
|
||||
prompt_ids = self.info.get_tokenizer().encode(
|
||||
prompt, add_special_tokens=False
|
||||
)
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt,
|
||||
mm_data,
|
||||
mm_kwargs,
|
||||
)
|
||||
if not mm_data:
|
||||
return processed_data
|
||||
|
||||
images = mm_data.get("images", [])
|
||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = [
|
||||
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
||||
]
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
num_patches = [
|
||||
self.info.get_num_patches(
|
||||
image_width=size.width,
|
||||
image_height=size.height,
|
||||
processor=hf_processor,
|
||||
mm_kwargs=mm_kwargs,
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
for size in image_sizes
|
||||
]
|
||||
processed_outputs["num_patches"] = torch.tensor(num_patches)
|
||||
processed_data["num_patches"] = torch.tensor(num_patches)
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -39,20 +39,19 @@ _I = TypeVar("_I", bound=Mistral3ProcessingInfo)
|
||||
|
||||
|
||||
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
# NOTE: LightOnOCR does not use break/end tokens, so we remove them here.
|
||||
input_ids = processed_outputs.get("input_ids")
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
return processed_data
|
||||
|
||||
input_ids = processed_data.get("input_ids")
|
||||
if input_ids is not None:
|
||||
processor = self.info.get_hf_processor()
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
@@ -67,22 +66,22 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
|
||||
torch.tensor([break_id, end_id]),
|
||||
)
|
||||
|
||||
processed_outputs["input_ids"] = input_ids[keep_mask].unsqueeze(0)
|
||||
if "attention_mask" in processed_outputs:
|
||||
processed_outputs["attention_mask"] = processed_outputs[
|
||||
"attention_mask"
|
||||
][keep_mask].unsqueeze(0)
|
||||
processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0)
|
||||
if "attention_mask" in processed_data:
|
||||
processed_data["attention_mask"] = processed_data["attention_mask"][
|
||||
keep_mask
|
||||
].unsqueeze(0)
|
||||
|
||||
# un-pad pixel_values per-image so caches remain independent.
|
||||
pixel_values = processed_outputs.get("pixel_values")
|
||||
pixel_values = processed_data.get("pixel_values")
|
||||
if pixel_values is not None:
|
||||
image_sizes = processed_outputs["image_sizes"]
|
||||
image_sizes = processed_data["image_sizes"]
|
||||
assert len(pixel_values) == len(image_sizes)
|
||||
processed_outputs["pixel_values"] = [
|
||||
processed_data["pixel_values"] = [
|
||||
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
|
||||
]
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -321,30 +321,30 @@ class PixtralHFProcessingInfo(BaseLlavaProcessingInfo):
|
||||
|
||||
|
||||
class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
pixel_values = processed_outputs.get("pixel_values")
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
return processed_data
|
||||
|
||||
pixel_values = processed_data.get("pixel_values")
|
||||
if pixel_values is not None:
|
||||
# Avoid padding since we need the output for each image to be
|
||||
# independent of other images for the cache to work correctly
|
||||
image_sizes = processed_outputs["image_sizes"]
|
||||
image_sizes = processed_data["image_sizes"]
|
||||
assert len(pixel_values) == len(image_sizes)
|
||||
|
||||
processed_outputs["pixel_values"] = [
|
||||
processed_data["pixel_values"] = [
|
||||
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
|
||||
]
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -211,6 +211,9 @@ class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]):
|
||||
|
||||
|
||||
class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -321,22 +321,33 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
pixel_values_videos=MultiModalFieldConfig.batched("video"),
|
||||
)
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
videos = mm_data.pop("videos", [])
|
||||
assert isinstance(videos, list)
|
||||
|
||||
if not videos:
|
||||
return super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
# LLaVA-OneVision processor doesn't support multiple videos
|
||||
# with different sizes when converting back to tensors
|
||||
@@ -346,19 +357,13 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
image_token = processor.image_token
|
||||
video_token = processor.video_token
|
||||
|
||||
text_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data={},
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
|
||||
images = mm_data.pop("images", [])
|
||||
assert isinstance(images, list)
|
||||
if images:
|
||||
processor_outputs = super()._call_hf_processor(
|
||||
prompt=image_token * len(images),
|
||||
mm_data={"images": images},
|
||||
mm_kwargs=mm_kwargs,
|
||||
processor_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=image_token * len(images), **{"images": images}),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
image_outputs = {
|
||||
k: v
|
||||
@@ -370,10 +375,10 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
|
||||
pixel_values_videos = []
|
||||
for video in videos:
|
||||
item_outputs = super()._call_hf_processor(
|
||||
prompt=video_token,
|
||||
mm_data={"videos": video},
|
||||
mm_kwargs=mm_kwargs,
|
||||
item_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=video_token, **{"videos": video}),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
|
||||
@@ -381,11 +386,13 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
video_outputs = {"pixel_values_videos": pixel_values_videos}
|
||||
|
||||
combined_outputs = dict(
|
||||
text_outputs,
|
||||
input_ids=prompt_text,
|
||||
**image_outputs,
|
||||
**video_outputs,
|
||||
)
|
||||
return BatchFeature(combined_outputs)
|
||||
processed_data = BatchFeature(combined_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -286,7 +286,7 @@ def _validate_video_sources(paths, model_config) -> list[str]:
|
||||
# participates in the standard decode-stage pipeline.
|
||||
#
|
||||
# * codec backend: NOT a loader. OV2's codec path needs the video path string
|
||||
# to survive into ``_call_hf_processor``, where the HF processor builds the
|
||||
# to survive into ``_apply_hf_processor_main``, where the HF processor builds the
|
||||
# codec canvas + smart_resize + patchify
|
||||
# (pixel_values/image_grid_thw/patch_positions). That transform is
|
||||
# path-level and inseparable; it cannot be reconstructed from pre-decoded
|
||||
@@ -302,7 +302,7 @@ def prepare_codec_video_input(video_path: str) -> tuple:
|
||||
|
||||
Returns ``(dummy_ndarray, metadata)`` where the ndarray satisfies the
|
||||
parser's 4-D shape check and the metadata carries the actual path to
|
||||
our ``_call_hf_processor``. Use as::
|
||||
our ``_apply_hf_processor_main``. Use as::
|
||||
|
||||
multi_modal_data = {"video": prepare_codec_video_input("foo.mp4")}
|
||||
|
||||
@@ -1283,7 +1283,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
|
||||
# ``video_needs_metadata=True`` makes the parser preserve both the
|
||||
# ``(frames, metadata)`` tuples from the frame backend and the
|
||||
# ``(dummy, {marker: path})`` tuples from prepare_codec_video_input;
|
||||
# both are dispatched by metadata content in ``_call_hf_processor``.
|
||||
# both are dispatched by metadata content in ``_apply_hf_processor_main``.
|
||||
return LlavaOnevision2MultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
@@ -1547,18 +1547,23 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
|
||||
class LlavaOnevision2MultiModalProcessor(
|
||||
BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# The wrapped OV2 processor is a bare custom class without the standard
|
||||
# ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path
|
||||
# fails; overriding this method routes the base class to call us
|
||||
# directly.
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
merged_kwargs.setdefault("return_tensors", "pt")
|
||||
call_kwargs = {
|
||||
k: v
|
||||
@@ -1627,21 +1632,25 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
if len(codec_video_paths) > 1
|
||||
else codec_video_paths[0]
|
||||
)
|
||||
# Route through the base ``_call_hf_processor`` so float-tensor
|
||||
# Route through ``ctx.call_hf_processor`` so float-tensor
|
||||
# dtype postprocessing is applied automatically; inject
|
||||
# ``video_backend="codec"`` via mm_kwargs so the wrapped processor
|
||||
# dispatches to its codec branch.
|
||||
output = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs={**mm_kwargs, "video_backend": "codec"},
|
||||
output = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(
|
||||
**{**hf_processor_mm_kwargs, "video_backend": "codec"}
|
||||
),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
{**hf_processor_mm_kwargs, "video_backend": "codec"},
|
||||
)
|
||||
data = dict(output)
|
||||
return BatchFeature(
|
||||
processed_data = BatchFeature(
|
||||
self._rename_codec_outputs_to_video(
|
||||
data, codec_video_paths, hf_processor
|
||||
)
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
# ---- Frame backend (registered LlavaOnevision2VideoBackend) ------
|
||||
# Every non-codec video reaches here as a ``(frames_ndarray, metadata)``
|
||||
@@ -1657,7 +1666,9 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
# local-file gating is enforced by the connector before decoding.
|
||||
if videos_present:
|
||||
timestamp_decimals = int(
|
||||
mm_kwargs.get("timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS)
|
||||
hf_processor_mm_kwargs.get(
|
||||
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
|
||||
)
|
||||
)
|
||||
|
||||
per_video_frames: list[list[Image.Image]] = []
|
||||
@@ -1671,7 +1682,7 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
# ``<{t} seconds><|vision_start|><|image_pad|><|vision_end|>``
|
||||
# blocks (matches the OV2 hf-chat reference exactly).
|
||||
new_prompt = _expand_video_markers_in_prompt(
|
||||
prompt,
|
||||
prompt_text,
|
||||
per_video_timestamps,
|
||||
timestamp_decimals=timestamp_decimals,
|
||||
)
|
||||
@@ -1700,7 +1711,7 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
row_is_video: list[bool] = []
|
||||
vid_idx = 0
|
||||
img_idx = 0
|
||||
for marker in marker_pattern.finditer(prompt):
|
||||
for marker in marker_pattern.finditer(prompt_text):
|
||||
if marker.lastgroup == "video":
|
||||
frames = per_video_frames[vid_idx]
|
||||
vid_idx += 1
|
||||
@@ -1723,15 +1734,15 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
merged_mm_data.pop("videos", None)
|
||||
merged_mm_data["images"] = flat_frames
|
||||
|
||||
# Route through the base ``_call_hf_processor`` (applies float-tensor
|
||||
# Route through ``ctx.call_hf_processor`` (applies float-tensor
|
||||
# dtype postprocessing automatically). The wrapped processor's image
|
||||
# branch ignores video/codec-only kwargs and does not forward extra
|
||||
# **kwargs to the image processor, so passing the full merged kwarg
|
||||
# set here is a no-op beyond return_tensors/padding.
|
||||
output = super()._call_hf_processor(
|
||||
prompt=new_prompt,
|
||||
mm_data=merged_mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
output = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=new_prompt, **merged_mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
data = dict(output)
|
||||
|
||||
@@ -1788,17 +1799,21 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
(len(per_video_timestamps),), dtype=torch.long
|
||||
)
|
||||
|
||||
return BatchFeature(data)
|
||||
processed_data = BatchFeature(data)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
# ---- Image-only / text-only call --------------------------------
|
||||
# No videos present: delegate to the base ``_call_hf_processor``, which
|
||||
# No videos present: delegate to ``ctx.call_hf_processor``, which
|
||||
# runs the wrapped processor over the (possibly empty) image set and
|
||||
# applies float-tensor dtype postprocessing automatically.
|
||||
return super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _rename_codec_outputs_to_video(
|
||||
self,
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
import collections
|
||||
import collections.abc
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Any, TypeAlias, cast
|
||||
from typing import Annotated, TypeAlias, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -583,15 +583,17 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
|
||||
class MiDashengLMMultiModalProcessor(
|
||||
BaseMultiModalProcessor[MiDashengLMProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, Any],
|
||||
) -> BatchFeature:
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
|
||||
# + Padding
|
||||
min_audio_len = self.info.get_min_audio_len()
|
||||
processed_audios = [
|
||||
np.pad(
|
||||
@@ -608,20 +610,7 @@ class MiDashengLMMultiModalProcessor(
|
||||
if processed_audios:
|
||||
mm_data["audio"] = processed_audios
|
||||
|
||||
if not mm_data.get("audio", []):
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
)
|
||||
|
||||
return super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -912,16 +912,23 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
||||
fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")
|
||||
return fields
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.
|
||||
Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses
|
||||
the singular form.
|
||||
"""
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
# Remap audios → audio (MiMoOmniProcessor uses singular param name)
|
||||
if "audios" in mm_data:
|
||||
mm_data = {**mm_data, "audio": mm_data["audios"]}
|
||||
@@ -999,7 +1006,13 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
||||
|
||||
mm_data = {**mm_data, "videos": converted}
|
||||
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -448,7 +448,7 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
||||
if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems):
|
||||
audio_inputs = {}
|
||||
else:
|
||||
audio_inputs = self._base_call_hf_processor(
|
||||
audio_inputs = self._call_hf_processor_on_prompts(
|
||||
prompts=[self.info.audio_pattern] * len(parsed_audios),
|
||||
mm_data={"audios": [[audio] for audio in parsed_audios]},
|
||||
mm_kwargs={**mm_kwargs, "chunk_input": True},
|
||||
|
||||
@@ -881,7 +881,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
|
||||
image_inputs = {}
|
||||
else:
|
||||
image_inputs = self._base_call_hf_processor(
|
||||
image_inputs = self._call_hf_processor_on_prompts(
|
||||
prompts=[self.info.image_pattern] * len(parsed_images),
|
||||
mm_data={"images": [[image] for image in parsed_images]},
|
||||
mm_kwargs=mm_kwargs,
|
||||
@@ -906,7 +906,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
|
||||
video_inputs = {}
|
||||
else:
|
||||
video_inputs = self._base_call_hf_processor(
|
||||
video_inputs = self._call_hf_processor_on_prompts(
|
||||
prompts=[
|
||||
self.info.image_pattern * len(video) for video in parsed_videos
|
||||
],
|
||||
@@ -983,7 +983,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
return new_token_ids, placeholders
|
||||
|
||||
def _base_call_hf_processor(
|
||||
def _call_hf_processor_on_prompts(
|
||||
self,
|
||||
prompts: list[str],
|
||||
mm_data: Mapping[str, Sequence[object]],
|
||||
@@ -993,19 +993,19 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
) -> dict[str, NestedTensors]:
|
||||
# This processor supports zipping prompt and mm_data together
|
||||
if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}:
|
||||
inputs = super()._call_hf_processor(
|
||||
prompt=prompts, # type: ignore
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
inputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompts, **mm_data),
|
||||
mm_kwargs,
|
||||
)
|
||||
else:
|
||||
inputs = defaultdict[str, list[torch.Tensor]](list)
|
||||
|
||||
for i, prompt in enumerate(prompts):
|
||||
inputs_one = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data={k: v[i] for k, v in mm_data.items()},
|
||||
mm_kwargs=mm_kwargs,
|
||||
inputs_one = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **{k: v[i] for k, v in mm_data.items()}),
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
for k, v in inputs_one.items():
|
||||
@@ -1014,23 +1014,31 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
return {k: inputs[k] for k in out_keys}
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt)])
|
||||
mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs)
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
|
||||
mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
|
||||
|
||||
return BatchFeature(
|
||||
processed_data = BatchFeature(
|
||||
{
|
||||
"input_ids": input_ids,
|
||||
**mm_inputs,
|
||||
}
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -233,30 +233,30 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo])
|
||||
|
||||
|
||||
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
pixel_values = processed_outputs.get("pixel_values")
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
return processed_data
|
||||
|
||||
pixel_values = processed_data.get("pixel_values")
|
||||
if pixel_values is not None:
|
||||
# Avoid padding since we need the output for each image to be
|
||||
# independent of other images for the cache to work correctly
|
||||
image_sizes = processed_outputs["image_sizes"]
|
||||
image_sizes = processed_data["image_sizes"]
|
||||
assert len(pixel_values) == len(image_sizes)
|
||||
|
||||
processed_outputs["pixel_values"] = [
|
||||
processed_data["pixel_values"] = [
|
||||
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
|
||||
]
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -589,23 +589,23 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
|
||||
class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
return processed_data
|
||||
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_processor = processor.image_processor
|
||||
vision_config = self.info.get_hf_config().vision_config
|
||||
|
||||
if processed_outputs.get("pixel_values") is not None:
|
||||
if processed_data.get("pixel_values") is not None:
|
||||
assert "images" in mm_data, (
|
||||
"images expected to be in mm_data when pixel_values is present"
|
||||
)
|
||||
@@ -636,10 +636,10 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
|
||||
1 if r_h * r_w == 1 else 1 + r_h * r_w for (r_h, r_w) in aspect_ratios
|
||||
]
|
||||
|
||||
processed_outputs["aspect_ratios"] = torch.tensor(aspect_ratios)
|
||||
processed_outputs["patches_per_image"] = torch.tensor(patches_per_image)
|
||||
processed_data["aspect_ratios"] = torch.tensor(aspect_ratios)
|
||||
processed_data["patches_per_image"] = torch.tensor(patches_per_image)
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -1123,17 +1123,44 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
|
||||
|
||||
|
||||
class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||
processor = self.info.get_hf_processor()
|
||||
|
||||
# The chat template is already applied to the prompt tokens
|
||||
# Use message_format="none" to avoid applying it again
|
||||
# Prepend an empty space if `always_start_with_space` is True
|
||||
tokens = processor.get_tokens_input(
|
||||
self.info.get_tokenizer().decode(prompt),
|
||||
message_format="none",
|
||||
always_start_with_space=True,
|
||||
)
|
||||
|
||||
# Prepend a BOS token id to the tokens
|
||||
return self.info.ctx.call_hf_processor(
|
||||
processor.process,
|
||||
dict(tokens=tokens),
|
||||
)["input_ids"].tolist()
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
|
||||
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not processor_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
hf_processor.process,
|
||||
dict(text=prompt, **mm_data),
|
||||
mm_kwargs,
|
||||
dict(text=prompt_text, **processor_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
tokenizer = hf_processor.tokenizer
|
||||
@@ -1141,17 +1168,16 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
||||
|
||||
image_processor = hf_processor.image_processor
|
||||
|
||||
input_ids: torch.Tensor = processed_outputs.pop("input_ids")
|
||||
processed_outputs["input_ids"] = input_ids.unsqueeze(0)
|
||||
processed_data.pop("input_ids")
|
||||
|
||||
if (images := mm_data.get("images")) is not None:
|
||||
if (images := processor_data.get("images")) is not None:
|
||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = [
|
||||
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
||||
]
|
||||
|
||||
feat_is_patch = processed_outputs["image_input_idx"] >= 0
|
||||
feat_is_patch = processed_data["image_input_idx"] >= 0
|
||||
|
||||
tilings = [
|
||||
self.info.select_tiling(
|
||||
@@ -1165,35 +1191,12 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
||||
num_crops = torch.tensor(tilings).prod(-1) + 1
|
||||
assert num_crops.sum() == len(feat_is_patch)
|
||||
|
||||
processed_outputs["num_crops"] = num_crops
|
||||
processed_outputs["img_patch_id"] = image_patch_id
|
||||
processed_data["num_crops"] = num_crops
|
||||
processed_data["img_patch_id"] = image_patch_id
|
||||
|
||||
return processed_outputs
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
def _apply_hf_processor_tokens_only(
|
||||
self,
|
||||
prompt_tokens: list[int],
|
||||
) -> list[int]:
|
||||
processor = self.info.get_hf_processor()
|
||||
|
||||
# The chat template is already applied to the prompt tokens
|
||||
# Use message_format="none" to avoid applying it again
|
||||
# Prepend an empty space if `always_start_with_space` is True
|
||||
tokens = processor.get_tokens_input(
|
||||
self.info.get_tokenizer().decode(prompt_tokens),
|
||||
message_format="none",
|
||||
always_start_with_space=True,
|
||||
)
|
||||
|
||||
# Prepend a BOS token id to the tokens
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
processor.process,
|
||||
dict(tokens=tokens),
|
||||
)
|
||||
prompt_ids = processed_data.pop("input_ids").tolist()
|
||||
print(prompt_ids, len(prompt_ids))
|
||||
|
||||
return prompt_ids
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -1929,30 +1929,33 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
|
||||
|
||||
|
||||
class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
def _apply_hf_processor_tokens_only(
|
||||
self,
|
||||
prompt_tokens: list[int],
|
||||
) -> list[int]:
|
||||
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||
processor = self.info.get_hf_processor()
|
||||
tokenizer = processor.tokenizer
|
||||
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
|
||||
|
||||
if len(prompt_tokens) == 0 or prompt_tokens[0] != bos_token_id:
|
||||
if len(prompt) == 0 or prompt[0] != bos_token_id:
|
||||
# Prepend the bos token to the prompt tokens
|
||||
prompt_tokens = [bos_token_id] + prompt_tokens
|
||||
prompt = [bos_token_id] + prompt
|
||||
|
||||
return prompt_tokens
|
||||
return prompt
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
|
||||
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
mm_data = dict(processor_data)
|
||||
|
||||
hf_config = self.info.get_hf_config()
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature:
|
||||
res = hf_processor(text=text, images=images, videos=videos, **kwargs)
|
||||
@@ -1984,14 +1987,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
# NOTE: metadata.frames_indices indicates
|
||||
# the sampled frames indices of pre-sampled videos, which is
|
||||
# used to calculate the timestamps. Make sure that
|
||||
# do_sample_frames in mm_kwargs is false for presampled videos.
|
||||
# do_sample_frames in hf_processor_mm_kwargs is false for
|
||||
# presampled videos.
|
||||
|
||||
# NOTE: a copy of mm_kwargs is created to update do_sample_frames,
|
||||
# otherwise mm_hash for the object will be incorrect.
|
||||
video_mm_kwargs = dict(**mm_kwargs)
|
||||
# NOTE: a copy of hf_processor_mm_kwargs is created to update
|
||||
# do_sample_frames, otherwise mm_hash for the object will be
|
||||
# incorrect.
|
||||
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
|
||||
if "do_sample_frames" not in video_mm_kwargs:
|
||||
# molmo_utils already has "do_sample_frames" in
|
||||
# mm_kwargs, don't overwrite it.
|
||||
# hf_processor_mm_kwargs, don't overwrite it.
|
||||
video_mm_kwargs["do_sample_frames"] = metadata.get(
|
||||
"do_sample_frames", False
|
||||
)
|
||||
@@ -2015,7 +2020,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
input_ids = input_ids[:, 1:]
|
||||
|
||||
video_string = tokenizer.batch_decode(input_ids)[0]
|
||||
prompt = prompt.replace(VIDEO_PROMPT, video_string, 1)
|
||||
prompt_text = prompt_text.replace(VIDEO_PROMPT, video_string, 1)
|
||||
|
||||
video_grids = video_outputs.pop("video_grids")
|
||||
assert video_grids[:, 0].sum() == len(
|
||||
@@ -2054,10 +2059,10 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
else:
|
||||
all_video_outputs = dict()
|
||||
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
patched_call,
|
||||
dict(text=prompt, **mm_data),
|
||||
mm_kwargs,
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
if (images := mm_data.get("images")) is not None:
|
||||
@@ -2077,22 +2082,22 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
for image_size in image_sizes
|
||||
]
|
||||
num_crops = torch.tensor(tilings).prod(-1) + 1
|
||||
assert sum(num_crops) == len(processed_outputs["pixel_values"])
|
||||
assert sum(num_crops) == processed_outputs["image_num_crops"].sum().item()
|
||||
assert sum(num_crops) == len(processed_data["pixel_values"])
|
||||
assert sum(num_crops) == processed_data["image_num_crops"].sum().item()
|
||||
|
||||
image_grids = processed_outputs.pop("image_grids")
|
||||
image_grids = processed_data.pop("image_grids")
|
||||
image_num_pooled_patches = image_grids[:, :2].prod(dim=1) + image_grids[
|
||||
:, 2:
|
||||
].prod(dim=1)
|
||||
|
||||
processed_outputs["image_num_pooled_patches"] = image_num_pooled_patches
|
||||
n_patches = processed_outputs["pixel_values"].shape[1]
|
||||
processed_outputs["image_num_patches"] = (
|
||||
processed_outputs["image_num_crops"] * n_patches
|
||||
processed_data["image_num_pooled_patches"] = image_num_pooled_patches
|
||||
n_patches = processed_data["pixel_values"].shape[1]
|
||||
processed_data["image_num_patches"] = (
|
||||
processed_data["image_num_crops"] * n_patches
|
||||
)
|
||||
(
|
||||
processed_outputs["image_tokens"],
|
||||
processed_outputs["num_image_tokens"],
|
||||
processed_data["image_tokens"],
|
||||
processed_data["num_image_tokens"],
|
||||
) = build_flat_image_bool_length(
|
||||
image_grids,
|
||||
hf_config,
|
||||
@@ -2101,7 +2106,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
use_single_crop_start_token=hf_processor.use_single_crop_start_token,
|
||||
)
|
||||
|
||||
return BatchFeature({**processed_outputs, **all_video_outputs})
|
||||
processed_data.update(all_video_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
processed_data.pop("input_ids")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -942,16 +942,6 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
|
||||
image_placeholder: str = "<image>"
|
||||
bos_image_placeholder: str = "<|endoftext|><image>"
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Moondream3's processor handles images directly rather than exposing a
|
||||
# separate `image_processor`, so keep the cache path on text+MM calls.
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
@cached_property
|
||||
def bos_image_placeholder_tokens(self) -> list[int]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
@@ -1335,23 +1335,32 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo
|
||||
|
||||
|
||||
class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
if audios:
|
||||
mm_data["audio"] = audios
|
||||
mm_kwargs = dict(mm_kwargs)
|
||||
processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs)
|
||||
return self.info.ctx.call_hf_processor(
|
||||
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||
processor_kwargs = _filter_moss_audio_processor_config(hf_processor_mm_kwargs)
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**processor_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
mm_data,
|
||||
{},
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -461,24 +461,29 @@ class MossTranscribeDiarizeDummyInputsBuilder(
|
||||
class MossTranscribeDiarizeMultiModalProcessor(
|
||||
BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
audios = _get_audios_from_mm_data(mm_data)
|
||||
if not audios:
|
||||
input_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
return BatchFeature({"input_ids": [input_ids]}, tensor_type="pt")
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
processed = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, audio=audios),
|
||||
mm_kwargs,
|
||||
)
|
||||
return _add_vllm_audio_metadata(processed, len(audios))
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
audios = _get_audios_from_mm_data(mm_data)
|
||||
|
||||
return dict(audio=audios), hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
audios = _get_audios_from_mm_data(mm_data)
|
||||
if audios:
|
||||
processed_data = _add_vllm_audio_metadata(processed_data, len(audios))
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -262,13 +262,19 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
|
||||
class MuseGlimmerMultiModalProcessor(
|
||||
BaseMultiModalProcessor[MuseGlimmerProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = processor.tokenizer
|
||||
config = self.info.get_hf_config()
|
||||
images = mm_data.get("images", ())
|
||||
@@ -276,7 +282,7 @@ class MuseGlimmerMultiModalProcessor(
|
||||
if not isinstance(images, Sequence) or not isinstance(videos, Sequence):
|
||||
raise TypeError("MuseGlimmer multi-modal data must be a sequence")
|
||||
|
||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
prompt_ids = tokenizer.encode(prompt_text, add_special_tokens=False)
|
||||
image_sentinel_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)
|
||||
video_sentinel_id = tokenizer.convert_tokens_to_ids(VIDEO_TOKEN)
|
||||
if prompt_ids.count(image_sentinel_id) != len(images):
|
||||
@@ -354,7 +360,9 @@ class MuseGlimmerMultiModalProcessor(
|
||||
video_pixel_values=video_pixels,
|
||||
video_feature_sizes=torch.tensor(video_sizes),
|
||||
)
|
||||
return BatchFeature(data=data, tensor_type=None)
|
||||
processed_data = BatchFeature(data=data, tensor_type=None)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -361,18 +361,8 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):
|
||||
class NanoNemotronVLMultiModalProcessor(
|
||||
BaseMultiModalProcessor[NanoNemotronVLProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Bypass `call_hf_processor_mm_only` by no-op overriding`_call_hf_processor`,
|
||||
so it chooses this path:
|
||||
`type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor`
|
||||
"""
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _get_image_fields_config(self, hf_inputs: BatchFeature):
|
||||
if self.info.is_dynamic_tiler:
|
||||
@@ -406,7 +396,9 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
)
|
||||
|
||||
def _get_audio_fields_config(self, hf_inputs: BatchFeature):
|
||||
audio_num_clips = torch.as_tensor(hf_inputs["audio_num_clips"])
|
||||
audio_num_clips = torch.as_tensor(
|
||||
hf_inputs.get("audio_num_clips", torch.empty(0))
|
||||
)
|
||||
|
||||
return dict(
|
||||
input_audio_features=MultiModalFieldConfig.flat_from_sizes(
|
||||
@@ -733,11 +725,8 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
# Bypass the cached path: the HF processor must receive the
|
||||
# prompt (with injected <so_embedding>) and the audio data
|
||||
# together so it can perform audio-token replacement natively.
|
||||
(
|
||||
prompt_ids,
|
||||
mm_info,
|
||||
is_update_applied,
|
||||
) = self._apply_hf_processor(inputs, timing_ctx)
|
||||
mm_info = self._apply_hf_processor(inputs, timing_ctx)
|
||||
prompt_ids = self._postprocess_prompt(inputs.prompt)
|
||||
|
||||
with timing_ctx.record("apply_prompt_updates"):
|
||||
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
|
||||
@@ -745,7 +734,7 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
prompt_ids=prompt_ids,
|
||||
mm_kwargs=mm_info.kwargs,
|
||||
mm_prompt_updates=mm_info.prompt_updates,
|
||||
is_update_applied=is_update_applied,
|
||||
is_update_applied=self.hf_processor_applies_updates,
|
||||
)
|
||||
|
||||
mm_placeholder_ranges = {
|
||||
|
||||
@@ -411,22 +411,6 @@ class NemotronParseMultiModalProcessor(
|
||||
) -> list[int]:
|
||||
return [0]
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
else:
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
tokenizer = hf_processor.tokenizer
|
||||
processed_outputs = tokenizer(
|
||||
prompt, add_special_tokens=False, return_tensors="pt"
|
||||
)
|
||||
return processed_outputs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -341,41 +341,30 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
|
||||
# -300 is image_atom token, filter them out
|
||||
return [vte_vocab_size + x + 300 for x in image_indicators if x < -300]
|
||||
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
# Avoid warning from HF logger for text-only input
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
return processed_data
|
||||
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
image_indicators = [
|
||||
hf_processor.construct_image_indicators(grid)
|
||||
for grid in processed_outputs["grids"]
|
||||
for grid in processed_data["grids"]
|
||||
]
|
||||
indicator_tokens = [
|
||||
self.image_indicators_to_visual_tokens(indicator)
|
||||
for indicator in image_indicators
|
||||
]
|
||||
processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens)
|
||||
return processed_outputs
|
||||
processed_data["indicator_tokens"] = torch.tensor(indicator_tokens)
|
||||
|
||||
def _apply_hf_processor_tokens_only(
|
||||
self,
|
||||
prompt_tokens: list[int],
|
||||
) -> list[int]:
|
||||
return prompt_tokens
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -332,53 +332,43 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
|
||||
if x >= INDICATOR_IDS[0]
|
||||
]
|
||||
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
# Avoid warning from HF logger for text-only input
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
return processed_data
|
||||
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
|
||||
if "videos" in mm_data:
|
||||
visual_indicators = [
|
||||
hf_processor.construct_visual_indicators((1, 1, 1), True)
|
||||
for grid in processed_outputs["video_grids"]
|
||||
for grid in processed_data["video_grids"]
|
||||
]
|
||||
indicator_tokens = [
|
||||
self.visual_indicators_to_visual_tokens(indicator)
|
||||
for indicator in visual_indicators
|
||||
]
|
||||
processed_outputs["video_indicator_tokens"] = torch.tensor(indicator_tokens)
|
||||
processed_data["video_indicator_tokens"] = torch.tensor(indicator_tokens)
|
||||
if "images" in mm_data:
|
||||
visual_indicators = [
|
||||
hf_processor.construct_visual_indicators((1, 1, 1), False)
|
||||
for grid in processed_outputs["grids"]
|
||||
for grid in processed_data["grids"]
|
||||
]
|
||||
indicator_tokens = [
|
||||
self.visual_indicators_to_visual_tokens(indicator)
|
||||
for indicator in visual_indicators
|
||||
]
|
||||
|
||||
processed_outputs["indicator_tokens"] = torch.tensor(indicator_tokens)
|
||||
return processed_outputs
|
||||
processed_data["indicator_tokens"] = torch.tensor(indicator_tokens)
|
||||
|
||||
def _apply_hf_processor_tokens_only(
|
||||
self,
|
||||
prompt_tokens: list[int],
|
||||
) -> list[int]:
|
||||
return prompt_tokens
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -244,32 +244,37 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing
|
||||
class PaddleOCRVLMultiModalProcessor(
|
||||
BaseMultiModalProcessor[PaddleOCRVLProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
final_mm_kwargs = dict(mm_kwargs or {})
|
||||
final_mm_kwargs.setdefault("images_kwargs", {})
|
||||
# vLLM use PIL.Image, always set channel_last
|
||||
final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**final_mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
mm_kwargs,
|
||||
)
|
||||
num_patches_per_image = processed_outputs["image_grid_thw"].prod(-1)
|
||||
processed_outputs["pixel_values"] = processed_outputs["pixel_values"].split(
|
||||
num_patches_per_image.tolist()
|
||||
)
|
||||
else:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
processed_outputs = tokenizer(
|
||||
prompt, add_special_tokens=True, return_tensors="pt"
|
||||
)
|
||||
return processed_outputs
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
final_mm_kwargs = dict(hf_processor_mm_kwargs or {})
|
||||
final_mm_kwargs.setdefault("images_kwargs", {})
|
||||
# vLLM use PIL.Image, always set channel_last
|
||||
final_mm_kwargs["input_data_format"] = ChannelDimension.LAST
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**final_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
num_patches_per_image = processed_data["image_grid_thw"].prod(-1)
|
||||
processed_data["pixel_values"] = processed_data["pixel_values"].split(
|
||||
num_patches_per_image.tolist()
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -152,23 +152,6 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo
|
||||
|
||||
|
||||
class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
if not mm_data:
|
||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
|
||||
return super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -396,19 +396,19 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
|
||||
|
||||
|
||||
class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
input_ids = processed_outputs["input_ids"]
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
return processed_data
|
||||
|
||||
input_ids = processed_data["input_ids"]
|
||||
assert isinstance(input_ids, torch.Tensor)
|
||||
|
||||
# Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
|
||||
@@ -416,7 +416,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
# Therefore, we need to do an early replacement here
|
||||
input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID)
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -868,43 +868,52 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
|
||||
|
||||
|
||||
class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
sr = self.info.get_feature_extractor(**mm_kwargs).sampling_rate
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
sr = self.info.get_feature_extractor(**hf_processor_mm_kwargs).sampling_rate
|
||||
if audio_data := mm_data.get("audios", []):
|
||||
mm_data["audios"] = [(data, sr) for data in audio_data]
|
||||
|
||||
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
num_img_tokens = [
|
||||
self.info.get_num_image_tokens(
|
||||
image_width=img_size[0],
|
||||
image_height=img_size[1],
|
||||
processor=hf_processor,
|
||||
)
|
||||
for img_size in processed_outputs["image_sizes"]
|
||||
for img_size in processed_data["image_sizes"]
|
||||
]
|
||||
processed_outputs["num_img_tokens"] = num_img_tokens
|
||||
processed_data["num_img_tokens"] = num_img_tokens
|
||||
|
||||
audio_features = processed_outputs["input_audio_embeds"]
|
||||
audio_features = processed_data["input_audio_embeds"]
|
||||
feature_sizes = [
|
||||
self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data
|
||||
]
|
||||
processed_outputs["input_audio_embeds"] = [
|
||||
processed_data["input_audio_embeds"] = [
|
||||
audio_features[idx, :size] for idx, size in enumerate(feature_sizes)
|
||||
]
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -142,16 +142,22 @@ class Phi4SiglipDummyInputsBuilder(
|
||||
class Phi4SiglipMultiModalProcessor(
|
||||
BaseMultiModalProcessor[Phi4SiglipProcessingInfo],
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
processed = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# The HF processor's tokenizer_image_token() replaces the "<image>"
|
||||
@@ -163,10 +169,12 @@ class Phi4SiglipMultiModalProcessor(
|
||||
# NOTE: tokenizer.__call__() (not .encode()) must be used so that
|
||||
# added/special tokens like <|user|>, <|end|> are kept as single IDs.
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
new_ids = tokenizer(prompt).input_ids
|
||||
new_ids = tokenizer(prompt_text).input_ids
|
||||
processed["input_ids"] = torch.tensor([new_ids])
|
||||
|
||||
return processed
|
||||
processed_data = processed
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -51,12 +51,10 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
MultiModalProcessingInfo,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
TimingContext,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -213,6 +211,9 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
|
||||
|
||||
|
||||
class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]):
|
||||
# The tokens are already inserted by the chat template
|
||||
hf_processor_applies_updates = True
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: Mapping[str, NestedTensors],
|
||||
@@ -220,24 +221,35 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
return dict(images=MultiModalFieldConfig.batched("image"))
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
# Avoid padding issue
|
||||
dict(**mm_kwargs, return_tensors=None),
|
||||
dict(**hf_processor_mm_kwargs, return_tensors=None),
|
||||
)
|
||||
|
||||
# Missing batch dimension
|
||||
if is_list_of(outputs["input_ids"], int):
|
||||
outputs["input_ids"] = [outputs["input_ids"]]
|
||||
|
||||
return outputs
|
||||
processed_data = outputs
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
@@ -273,16 +285,6 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
|
||||
),
|
||||
]
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
||||
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
# NOTE: The tokens are already inserted by the chat template
|
||||
return prompt_ids, mm_info, True
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
PixtralMultiModalProcessor,
|
||||
|
||||
@@ -499,83 +499,6 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
|
||||
class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
|
||||
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
|
||||
if audios:
|
||||
# NOTE: Qwen2.5-Omni processor accept "audio"
|
||||
mm_data["audio"] = audios
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
)
|
||||
elif (
|
||||
mm_kwargs.get("use_audio_in_video")
|
||||
and "audio" not in mm_data
|
||||
and mm_data.get("videos")
|
||||
):
|
||||
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
|
||||
# and populated mm_data["audio"] itself before delegating here,
|
||||
# in which case this isn't a genuine "no audio" case. Likewise,
|
||||
# mm_data can be empty on a multimodal-processor-cache hit, where
|
||||
# there's nothing to (re-)process this call and the real result
|
||||
# comes from the cache — not a genuine "no audio" case either.
|
||||
raise ValueError(
|
||||
"Video doesn't have audio track with `audio_in_video=True`"
|
||||
)
|
||||
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
|
||||
if mm_data.get("videos") and (
|
||||
merged.keys() & {"size", "min_pixels", "max_pixels"}
|
||||
):
|
||||
mm_kwargs = dict(mm_kwargs)
|
||||
video_size = dict(self.info.get_hf_processor().video_processor.size)
|
||||
size_override = merged.get("size")
|
||||
if size_override is not None:
|
||||
video_size = video_size | size_override
|
||||
min_pixels = merged.get("min_pixels")
|
||||
if min_pixels is not None:
|
||||
video_size["shortest_edge"] = min_pixels
|
||||
max_pixels = merged.get("max_pixels")
|
||||
if max_pixels is not None:
|
||||
video_size["longest_edge"] = max_pixels
|
||||
mm_kwargs["size"] = video_size
|
||||
|
||||
hf_inputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
|
||||
input_features = hf_inputs.pop("input_features", None)
|
||||
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
|
||||
if "input_audio_features" not in hf_inputs and input_features is not None:
|
||||
if feature_attention_mask is not None:
|
||||
input_features = input_features.permute(0, 2, 1)[
|
||||
feature_attention_mask.bool()
|
||||
].permute(1, 0)
|
||||
hf_inputs["input_audio_features"] = input_features
|
||||
if (
|
||||
"audio_feature_lengths" not in hf_inputs
|
||||
and feature_attention_mask is not None
|
||||
):
|
||||
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
|
||||
|
||||
video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
|
||||
if video_second_per_grid is not None:
|
||||
hf_inputs["second_per_grid_ts"] = video_second_per_grid
|
||||
|
||||
use_audio_in_video = mm_kwargs.get("use_audio_in_video", False)
|
||||
hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
|
||||
|
||||
return hf_inputs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
@@ -885,7 +808,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
),
|
||||
]
|
||||
|
||||
def _apply_hf_processor_mm_only(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
@@ -903,13 +826,87 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
)
|
||||
mm_counts["audio"] -= mm_counts["video"]
|
||||
|
||||
_, mm_processed_data = self._apply_hf_processor_text_mm(
|
||||
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
mm_data = dict(processor_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
|
||||
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
|
||||
if audios:
|
||||
# NOTE: Qwen2.5-Omni processor accept "audio"
|
||||
mm_data["audio"] = audios
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
)
|
||||
elif (
|
||||
hf_processor_mm_kwargs.get("use_audio_in_video")
|
||||
and "audio" not in mm_data
|
||||
and mm_data.get("videos")
|
||||
):
|
||||
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
|
||||
# and populated mm_data["audio"] itself before delegating here,
|
||||
# in which case this isn't a genuine "no audio" case. Likewise,
|
||||
# mm_data can be empty on a multimodal-processor-cache hit, where
|
||||
# there's nothing to (re-)process this call and the real result
|
||||
# comes from the cache — not a genuine "no audio" case either.
|
||||
raise ValueError(
|
||||
"Video doesn't have audio track with `audio_in_video=True`"
|
||||
)
|
||||
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
if mm_data.get("videos") and (
|
||||
merged.keys() & {"size", "min_pixels", "max_pixels"}
|
||||
):
|
||||
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||
video_size = dict(self.info.get_hf_processor().video_processor.size)
|
||||
size_override = merged.get("size")
|
||||
if size_override is not None:
|
||||
video_size = video_size | size_override
|
||||
min_pixels = merged.get("min_pixels")
|
||||
if min_pixels is not None:
|
||||
video_size["shortest_edge"] = min_pixels
|
||||
max_pixels = merged.get("max_pixels")
|
||||
if max_pixels is not None:
|
||||
video_size["longest_edge"] = max_pixels
|
||||
hf_processor_mm_kwargs["size"] = video_size
|
||||
|
||||
hf_inputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
return mm_processed_data
|
||||
input_features = hf_inputs.pop("input_features", None)
|
||||
feature_attention_mask = hf_inputs.get("feature_attention_mask", None)
|
||||
if "input_audio_features" not in hf_inputs and input_features is not None:
|
||||
if feature_attention_mask is not None:
|
||||
input_features = input_features.permute(0, 2, 1)[
|
||||
feature_attention_mask.bool()
|
||||
].permute(1, 0)
|
||||
hf_inputs["input_audio_features"] = input_features
|
||||
if (
|
||||
"audio_feature_lengths" not in hf_inputs
|
||||
and feature_attention_mask is not None
|
||||
):
|
||||
hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)
|
||||
|
||||
video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
|
||||
if video_second_per_grid is not None:
|
||||
hf_inputs["second_per_grid_ts"] = video_second_per_grid
|
||||
|
||||
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
|
||||
hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
|
||||
|
||||
processed_data = hf_inputs
|
||||
processed_data.update(passthrough_data)
|
||||
processed_data.pop("input_ids")
|
||||
|
||||
return processed_data
|
||||
|
||||
|
||||
class Qwen2_5OmniConditionalGenerationMixin:
|
||||
|
||||
@@ -1154,16 +1154,6 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
||||
)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override to use the text path instead of token path to use the
|
||||
# video-specific logic in processing_qwen2_5_vl.py
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -235,36 +235,27 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
|
||||
|
||||
|
||||
class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, Any],
|
||||
) -> BatchFeature:
|
||||
# NOTE - we rename audios -> audio in mm data because transformers has
|
||||
# deprecated audios for the qwen2audio processor and will remove
|
||||
# support for it in transformers 4.54.
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
if audios:
|
||||
mm_data["audio"] = audios
|
||||
|
||||
# Text-only input not supported in composite processor
|
||||
if not mm_data.get("audio", []):
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
|
||||
return super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -1195,79 +1195,95 @@ Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder
|
||||
class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
Qwen2_5OmniThinkerMultiModalProcessor,
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _get_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
||||
|
||||
def pad_to_hop_length(x: np.ndarray, hop_length: int) -> np.ndarray:
|
||||
length = x.shape[-1]
|
||||
if length % hop_length != 0:
|
||||
pad_length = hop_length - (length % hop_length)
|
||||
x = np.pad(x, (0, pad_length), mode="constant", constant_values=0)
|
||||
return x
|
||||
|
||||
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
hop_length = feature_extractor.hop_length
|
||||
audios = processor_data.get("audios")
|
||||
if audios:
|
||||
# NOTE: Qwen3-Omni processor accept "audio"
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
hop_length = feature_extractor.hop_length
|
||||
|
||||
def pad_to_hop_length(x: np.ndarray) -> np.ndarray:
|
||||
length = x.shape[-1]
|
||||
if length % hop_length != 0:
|
||||
pad_length = hop_length - (length % hop_length)
|
||||
x = np.pad(x, (0, pad_length), mode="constant", constant_values=0)
|
||||
return x
|
||||
|
||||
# To make sure the cache works with padding=True, we pre-padded
|
||||
# the audio to multiple of hop_length.
|
||||
mm_data["audio"] = [
|
||||
pad_to_hop_length(audio, hop_length)
|
||||
# the audio to a multiple of hop_length.
|
||||
processor_data = dict(processor_data)
|
||||
processor_data["audios"] = [
|
||||
pad_to_hop_length(audio)
|
||||
if isinstance(audio, np.ndarray)
|
||||
else (pad_to_hop_length(audio[0], hop_length), audio[1])
|
||||
else (pad_to_hop_length(audio[0]), audio[1])
|
||||
for audio in audios
|
||||
]
|
||||
|
||||
return processor_data, passthrough_data
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_items.get_all_counts().get("audio", 0):
|
||||
# TODO(Isotr0py): Remove this patch after upstream fix PR
|
||||
# released and Transformers version update:
|
||||
# https://github.com/huggingface/transformers/pull/41473
|
||||
mm_kwargs = dict(mm_kwargs)
|
||||
mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {})
|
||||
mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {})
|
||||
elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"):
|
||||
# mm_data can be empty on a multimodal-processor-cache hit, where
|
||||
# there's nothing to (re-)process this call and the real result
|
||||
# comes from the cache — not a genuine "no audio" case.
|
||||
raise ValueError(
|
||||
"Video doesn't have audio track with `audio_in_video=True`"
|
||||
hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)
|
||||
hf_processor_mm_kwargs["audio_kwargs"] = dict(
|
||||
hf_processor_mm_kwargs.get("audio_kwargs") or {}
|
||||
)
|
||||
hf_processor_mm_kwargs["text_kwargs"] = dict(
|
||||
hf_processor_mm_kwargs.get("text_kwargs") or {}
|
||||
)
|
||||
|
||||
hf_inputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
processed_data = super()._apply_hf_processor_main(
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
if (
|
||||
"audio_feature_lengths" in hf_inputs
|
||||
and "feature_attention_mask" in hf_inputs
|
||||
and (audios := mm_data.get("audio", []))
|
||||
"audio_feature_lengths" in processed_data
|
||||
and "feature_attention_mask" in processed_data
|
||||
):
|
||||
audio_num_frames = []
|
||||
for _, audio in enumerate(audios):
|
||||
audio_length = len(audio[0]) if isinstance(audio, tuple) else len(audio)
|
||||
num_frame = (
|
||||
(audio_length // hop_length)
|
||||
if audio_length % hop_length == 0
|
||||
else (audio_length // hop_length - 1)
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
processor_data, _ = self._get_hf_mm_data(valid_mm_items)
|
||||
audios = processor_data.get("audios", [])
|
||||
if audios:
|
||||
feature_extractor = self.info.get_feature_extractor(
|
||||
**hf_processor_mm_kwargs
|
||||
)
|
||||
if mm_kwargs.get("truncation", False):
|
||||
num_frame = min(
|
||||
num_frame, feature_extractor.n_samples // hop_length
|
||||
hop_length = feature_extractor.hop_length
|
||||
|
||||
audio_num_frames = []
|
||||
for audio in audios:
|
||||
audio_length = (
|
||||
len(audio[0]) if isinstance(audio, tuple) else len(audio)
|
||||
)
|
||||
audio_num_frames.append(num_frame)
|
||||
hf_inputs["feature_attention_mask"] = [
|
||||
torch.ones(num_frame) for num_frame in audio_num_frames
|
||||
]
|
||||
hf_inputs["audio_feature_lengths"] = torch.tensor(audio_num_frames)
|
||||
return hf_inputs
|
||||
num_frame = (
|
||||
(audio_length // hop_length)
|
||||
if audio_length % hop_length == 0
|
||||
else (audio_length // hop_length - 1)
|
||||
)
|
||||
if hf_processor_mm_kwargs.get("truncation", False):
|
||||
num_frame = min(
|
||||
num_frame, feature_extractor.n_samples // hop_length
|
||||
)
|
||||
audio_num_frames.append(num_frame)
|
||||
|
||||
processed_data["feature_attention_mask"] = [
|
||||
torch.ones(num_frame) for num_frame in audio_num_frames
|
||||
]
|
||||
processed_data["audio_feature_lengths"] = torch.tensor(audio_num_frames)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -1280,12 +1280,21 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
|
||||
return proc_impl is not None and proc_impl is not mixin_impl
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
|
||||
# Separate video processing from image processing. Because the videos
|
||||
@@ -1314,8 +1323,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
|
||||
# NOTE: a copy of is created to update do_sample_frames,
|
||||
# otherwise mm_hash for the object will be incorrect.
|
||||
video_mm_kwargs = dict(**mm_kwargs)
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
|
||||
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
|
||||
video_size = dict(self.info.get_video_processor().size)
|
||||
size_override = merged.get("size")
|
||||
@@ -1365,10 +1374,13 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
|
||||
video_mm_kwargs["fps"] = None
|
||||
|
||||
video_outputs = super()._call_hf_processor(
|
||||
prompt="<|vision_start|><|video_pad|><|vision_end|>",
|
||||
mm_data=video_mm_data,
|
||||
mm_kwargs=video_mm_kwargs,
|
||||
video_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**video_mm_kwargs),
|
||||
dict(
|
||||
text="<|vision_start|><|video_pad|><|vision_end|>",
|
||||
**video_mm_data,
|
||||
),
|
||||
video_mm_kwargs,
|
||||
)
|
||||
|
||||
# Discard HF output input_ids — we use get_video_repl below
|
||||
@@ -1424,12 +1436,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
# fps/num_frames are video-only kwargs already consumed by the loop;
|
||||
# exclude them so the text/image processor call below never gets a list.
|
||||
non_video_mm_kwargs = {
|
||||
k: v for k, v in mm_kwargs.items() if k not in ("fps", "num_frames")
|
||||
k: v
|
||||
for k, v in hf_processor_mm_kwargs.items()
|
||||
if k not in ("fps", "num_frames")
|
||||
}
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=non_video_mm_kwargs,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**non_video_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
non_video_mm_kwargs,
|
||||
)
|
||||
|
||||
# Replace each placeholder with pre-computed video tokens.
|
||||
@@ -1443,20 +1457,19 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
hf_config.video_token_id,
|
||||
hf_config.vision_end_token_id,
|
||||
]
|
||||
input_ids = processed_outputs.pop("input_ids")
|
||||
input_ids = processed_data.pop("input_ids")
|
||||
if not isinstance(input_ids, list):
|
||||
input_ids = input_ids.tolist()
|
||||
(prompt_ids,) = input_ids
|
||||
expanded_ids = _replace_video_token_placeholders(
|
||||
prompt_ids, video_target, video_input_ids_lst
|
||||
)
|
||||
processed_outputs["input_ids"] = [expanded_ids]
|
||||
processed_data["input_ids"] = [expanded_ids]
|
||||
|
||||
combined_outputs = dict(
|
||||
processed_outputs,
|
||||
**video_outputs,
|
||||
)
|
||||
return BatchFeature(combined_outputs)
|
||||
processed_data.update(video_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -345,28 +345,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
|
||||
return mm_fields
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Run the HF processor and unpad inputs."""
|
||||
try:
|
||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
except ValueError:
|
||||
if any(mm_data.values()):
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
hf_inputs = transformers.BatchFeature(
|
||||
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(hf_inputs)
|
||||
self._unpad_audios(hf_inputs, mm_data, mm_kwargs)
|
||||
return hf_inputs
|
||||
|
||||
def _unpad_audios(
|
||||
self,
|
||||
hf_inputs: "BatchFeature",
|
||||
@@ -450,19 +428,6 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
deriving them from updates."""
|
||||
return []
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Ask which modality owns each token of the expanded prompt, which is the
|
||||
only marking of the tokens an expansion adds around an item."""
|
||||
if any(mm_data.values()):
|
||||
mm_data = {**mm_data, "return_mm_token_type_ids": True}
|
||||
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
||||
"""Token ids marking where `modality` sits in the prompt, which for some
|
||||
processors differ from the placeholder written into it.
|
||||
@@ -612,24 +577,55 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
prompt = hf_processor.decode(prompt)
|
||||
prompt_text = hf_processor.decode(prompt)
|
||||
|
||||
# Bypass cached processor and always apply to the full set of mm inputs
|
||||
# NOTE: we can't just set caching=False because base class method
|
||||
# transforms outputs to `MultiModalKwargs` which is not going to
|
||||
# work for Transformers. The vision path has logic tied to
|
||||
# `mm_tokens_per_modality` in _apply_vision()
|
||||
prompt_ids, processed_data = self._apply_hf_processor_text_mm(
|
||||
prompt_text=prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={
|
||||
**hf_processor_mm_kwargs,
|
||||
# HF processors only accept text, and the decoded string already
|
||||
# contains any special tokens, so don't let them be added again
|
||||
# (`_call_hf_processor` disables `add_special_tokens`).
|
||||
"add_special_tokens": False,
|
||||
},
|
||||
hf_processor_mm_kwargs = {
|
||||
**hf_processor_mm_kwargs,
|
||||
# HF processors only accept text, and the decoded string already
|
||||
# contains any special tokens, so don't let them be added again
|
||||
# (the HF processor call disables `add_special_tokens`).
|
||||
"add_special_tokens": False,
|
||||
}
|
||||
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
# Ask which modality owns each token of the expanded prompt, which
|
||||
# is the only marking of the tokens an expansion adds around an item
|
||||
if any(processor_data.values()):
|
||||
processor_data = {**processor_data, "return_mm_token_type_ids": True}
|
||||
|
||||
try:
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **processor_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
except ValueError:
|
||||
if any(processor_data.values()):
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt_text)
|
||||
processed_data = transformers.BatchFeature(
|
||||
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(processed_data)
|
||||
self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
input_ids = processed_data.pop("input_ids")
|
||||
if not isinstance(input_ids, list):
|
||||
input_ids = input_ids.tolist()
|
||||
|
||||
(prompt_ids,) = input_ids
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
@@ -773,17 +769,41 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
except (AttributeError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Ask for the replacement each placeholder expands to, and record it as
|
||||
per-item fields: its token ids, and the tokens or patches behind them."""
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
|
||||
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
# Ask for the replacement each placeholder expands to, and record it as
|
||||
# per-item fields: its token ids, and the tokens or patches behind them
|
||||
if has_mm_data := any(mm_data.values()):
|
||||
mm_data = {**mm_data, "return_text_replacement_offsets": True}
|
||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
try:
|
||||
hf_inputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
except ValueError:
|
||||
if any(mm_data.values()):
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
hf_inputs = transformers.BatchFeature(
|
||||
dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(hf_inputs)
|
||||
self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)
|
||||
|
||||
# Drop the inputs the model would reject
|
||||
hf_inputs.pop("mm_token_type_ids", None)
|
||||
hf_inputs.pop("token_type_ids", None)
|
||||
@@ -802,6 +822,8 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"so it can be fixed, and install `transformers<5.15.0` in the "
|
||||
"meantime to locate placeholders in the expanded prompt instead."
|
||||
)
|
||||
hf_inputs.update(passthrough_data)
|
||||
hf_inputs.pop("input_ids", None)
|
||||
return hf_inputs
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
@@ -829,6 +851,9 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
|
||||
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
|
||||
|
||||
hf_inputs.update(passthrough_data)
|
||||
hf_inputs.pop("input_ids")
|
||||
|
||||
return hf_inputs
|
||||
|
||||
|
||||
|
||||
@@ -190,41 +190,84 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
|
||||
|
||||
|
||||
class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Text-only input not supported in composite processor
|
||||
if not mm_data.get("audios", []):
|
||||
prompt_ids = self.info.get_tokenizer().encode(
|
||||
prompt, add_special_tokens=False
|
||||
)
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
audios = mm_data.get("audios", [])
|
||||
assert isinstance(audios, list)
|
||||
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
include_audio_num_chunks=True,
|
||||
)
|
||||
|
||||
item_processor_data = dict(**mm_data, audios=audios)
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
output = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=item_processor_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
output["audio_features"] = output.pop("audio_values")
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
return output
|
||||
if processor_data:
|
||||
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
|
||||
if prompt_text is not None:
|
||||
processor_data = dict(text=prompt_text, **processor_data)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
def patched_call(**kwargs) -> BatchFeature:
|
||||
# The remote UltravoxProcessor does not support the
|
||||
# `truncation` kwarg injected by `call_hf_processor`: it
|
||||
# passes `truncation` to its audio processor by itself while
|
||||
# also forwarding `**kwargs` there, causing a duplicate
|
||||
# keyword argument error. Its hardcoded `truncation=False`
|
||||
# already matches vLLM's intended behavior.
|
||||
kwargs.pop("truncation", None)
|
||||
|
||||
return hf_processor(**kwargs)
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
patched_call,
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
else:
|
||||
processed_data = BatchFeature(dict(passthrough_data))
|
||||
|
||||
return self._postprocess_hf_mm_data(
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
processed_data,
|
||||
)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "audio_values" in processed_data:
|
||||
processed_data["audio_features"] = processed_data.pop("audio_values")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -225,7 +225,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
||||
) -> MultiModalProcessingInfo:
|
||||
# The processor logic differs for single-image (crop) vs multi-image
|
||||
# (no crop) requests. The processing cache assumes per-item output is
|
||||
# invariant of how many images are passed per prompt, so we only cache
|
||||
|
||||
@@ -47,12 +47,10 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
MultiModalProcessingInfo,
|
||||
PlaceholderFeaturesInfo,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
TimingContext,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
@@ -204,6 +202,9 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
|
||||
|
||||
|
||||
class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
|
||||
# The tokens are already inserted by the chat template
|
||||
hf_processor_applies_updates = True
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: Mapping[str, NestedTensors],
|
||||
@@ -220,12 +221,14 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
||||
# skip validation here
|
||||
pass
|
||||
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
|
||||
@@ -233,18 +236,23 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
||||
# MistralCommonVoxtralProcessor accepts "audio"
|
||||
mm_data["audio"] = audios
|
||||
|
||||
outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
# Avoid padding issue
|
||||
dict(**mm_kwargs, return_tensors=None),
|
||||
)
|
||||
# Avoid padding issue
|
||||
hf_processor_mm_kwargs = dict(**hf_processor_mm_kwargs, return_tensors=None)
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
# Missing batch dimension
|
||||
if is_list_of(outputs["input_ids"], int):
|
||||
outputs["input_ids"] = [outputs["input_ids"]]
|
||||
input_ids = processed_data.get("input_ids")
|
||||
if is_list_of(input_ids, int):
|
||||
processed_data["input_ids"] = [input_ids]
|
||||
|
||||
return outputs
|
||||
return processed_data
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
@@ -287,16 +295,6 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
||||
),
|
||||
]
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
||||
prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
# NOTE: The tokens are already inserted by the chat template
|
||||
return prompt_ids, mm_info, True
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
VoxtralMultiModalProcessor,
|
||||
|
||||
@@ -756,27 +756,36 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
|
||||
# for encoder profiling.
|
||||
return [0]
|
||||
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
mm_data = dict(audio=mm_data.pop("audios"))
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
if "labels" in processed_outputs:
|
||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
||||
return processed_outputs
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "labels" in processed_data:
|
||||
processed_data["input_ids"] = processed_data.pop("labels")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user