[Model] Use helper function to run MM processors with token inputs (where applicable) (#38018)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-03-26 16:44:04 +08:00
committed by GitHub
parent 52069012fe
commit 502c41a8f6
12 changed files with 215 additions and 145 deletions
+44 -1
View File
@@ -11,12 +11,16 @@ from transformers import (
AutoImageProcessor,
AutoProcessor,
AutoVideoProcessor,
BatchFeature,
processing_utils,
)
from transformers.audio_utils import AudioInput
from transformers.feature_extraction_utils import FeatureExtractionMixin
from transformers.image_processing_utils import BaseImageProcessor
from transformers.image_utils import ImageInput
from transformers.processing_utils import ProcessorMixin
from transformers.video_processing_utils import BaseVideoProcessor
from transformers.video_utils import VideoInput
from typing_extensions import TypeVar
from vllm.logger import init_logger
@@ -272,7 +276,6 @@ def get_processor_kwargs_keys(
"images_kwargs",
"videos_kwargs",
"audio_kwargs",
"common_kwargs",
}
try:
@@ -523,3 +526,43 @@ def cached_video_processor_from_config(
processor_cls_overrides=processor_cls, # type: ignore[arg-type]
**_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs),
)
def call_hf_processor_mm_only(
processor: ProcessorMixin,
images: ImageInput | None = None,
videos: VideoInput | None = None,
audio: AudioInput | None = None,
**kwargs,
) -> BatchFeature:
output_kwargs = processor._merge_kwargs(
get_processor_kwargs_type(processor),
**kwargs,
)
if audio is not None and (
feature_extractor := getattr(processor, "feature_extractor", None)
):
audio_inputs = feature_extractor(audio, **output_kwargs["audio_kwargs"])
audio_inputs["feature_attention_mask"] = audio_inputs.pop("attention_mask")
else:
audio_inputs = {}
if images is not None and (
image_processor := getattr(processor, "image_processor", None)
):
images_inputs = image_processor(images=images, **output_kwargs["images_kwargs"])
else:
images_inputs = {}
if videos is not None and (
video_processor := getattr(processor, "video_processor", None)
):
videos_inputs = video_processor(videos=videos, **output_kwargs["videos_kwargs"])
else:
videos_inputs = {}
return BatchFeature(
data={**audio_inputs, **images_inputs, **videos_inputs},
tensor_type=kwargs.get("return_tensors"),
)
+32 -16
View File
@@ -1,16 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import math
from typing import Any
from typing import Any, TypedDict
import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image
from transformers import BatchFeature, ProcessorMixin, TensorType
from typing_extensions import TypedDict, Unpack
from transformers.processing_utils import ProcessingKwargs
from typing_extensions import Unpack
from vllm.tokenizers.hf import HfTokenizer
@@ -308,15 +307,22 @@ def process_vision_for_patches(
return patches, dims_virtual
class IsaacImageProcessorKwargs(TypedDict, total=False):
class IsaacImagesKwargs(TypedDict, total=False):
patch_size: int
max_num_patches: int
min_num_patches: int
pixel_shuffle_scale: int
class IsaacProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
images_kwargs: IsaacImagesKwargs
_defaults = {
"text_kwargs": {"padding": False},
"images_kwargs": {},
}
class IsaacImageProcessor:
valid_kwargs = IsaacImageProcessorKwargs
model_input_names = ["pixel_values", "image_grid_thw"]
def __init__(
@@ -335,7 +341,7 @@ class IsaacImageProcessor:
self,
images: Image.Image | list[Image.Image],
return_tensors: str | TensorType | None = None,
**kwargs: Unpack[IsaacImageProcessorKwargs],
**kwargs: Unpack[IsaacImagesKwargs],
) -> BatchFeature:
"""Preprocess images into format compatible with vLLM input processing."""
if not isinstance(images, list):
@@ -349,10 +355,16 @@ class IsaacImageProcessor:
patches, dims_virtual = process_vision_for_patches(
image_tensor,
patch_size=self.patch_size,
max_num_patches=self.vision_max_num_patches,
min_num_patches=self.vision_min_num_patches,
pixel_shuffle_scale=self.pixel_shuffle_scale,
patch_size=kwargs.get("patch_size", self.patch_size),
max_num_patches=kwargs.get(
"max_num_patches", self.vision_max_num_patches
),
min_num_patches=kwargs.get(
"min_num_patches", self.vision_min_num_patches
),
pixel_shuffle_scale=kwargs.get(
"pixel_shuffle_scale", self.pixel_shuffle_scale
),
)
# Isaac packs a dummy temporal dim for images
@@ -405,13 +417,17 @@ class IsaacProcessor(ProcessorMixin):
text: str | list[str] | None = None,
images: Image.Image | list[Image.Image] | None = None,
return_tensors: str | TensorType | None = None,
**kwargs,
**kwargs: Unpack[IsaacProcessorKwargs], # type: ignore[misc]
) -> BatchFeature:
output_kwargs = self._merge_kwargs(
IsaacProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
if images is not None:
image_inputs = self.image_processor(
images,
return_tensors=return_tensors,
**kwargs,
images, **output_kwargs["images_kwargs"]
)
image_grid_thw = image_inputs["image_grid_thw"]
else:
@@ -435,7 +451,7 @@ class IsaacProcessor(ProcessorMixin):
index += 1
text[i] = text[i].replace("<|placeholder|>", "<|image_pad|>")
text_inputs = self.tokenizer(text, return_tensors=return_tensors)
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
else:
text_inputs = {}
+9 -58
View File
@@ -5,10 +5,7 @@ from mistral_common.protocol.instruct.chunk import ImageChunk
from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
from PIL import Image
from transformers import BatchFeature, ProcessorMixin, TensorType
from transformers.audio_utils import AudioInput
from transformers.image_utils import ImageInput
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
from transformers.video_utils import VideoInput
from vllm.tokenizers.mistral import MistralTokenizer
@@ -55,62 +52,16 @@ class MistralCommonPixtralProcessor(ProcessorMixin):
def __init__(self, tokenizer: MistralTokenizer) -> None:
self.tokenizer = tokenizer.transformers_tokenizer
# Back-compatibility for Transformers v4
if not hasattr(self.tokenizer, "init_kwargs"):
self.tokenizer.init_kwargs = {}
self.image_processor = MistralCommonImageProcessor(
tokenizer.instruct.mm_encoder
)
self._image_special_ids = self.image_processor.mm_encoder.special_ids
@property
def image_break_id(self) -> int:
return self._image_special_ids.img_break
@property
def image_token_id(self) -> int:
return self._image_special_ids.img
@property
def image_end_id(self) -> int:
return self._image_special_ids.img_end
def __call__(
self,
images: ImageInput | None = None,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput]
| None = None,
videos: VideoInput | None = None,
audio: AudioInput | None = None,
**kwargs,
):
if images is None and text is None and videos is None and audio is None:
raise ValueError(
f"You need to provide at least one input to "
f"call {self.__class__.__name__}"
)
kwargs = self._merge_kwargs(
self.valid_processor_kwargs,
tokenizer_init_kwargs={},
**kwargs,
)
kwargs["text_kwargs"]["return_tensors"] = "pt"
kwargs["images_kwargs"]["return_tensors"] = None # Avoid padding issue
attribute_to_kwargs = {
"tokenizer": (text, "text_kwargs"),
"image_processor": (images, "images_kwargs"),
"video_processor": (videos, "videos_kwargs"),
"feature_extractor": (audio, "audio_kwargs"),
}
outputs = {}
for attribute_name in self.attributes:
attribute = getattr(self, attribute_name, None)
input_data, input_kwargs = attribute_to_kwargs[attribute_name]
if input_data is not None and attribute is not None:
attribute_output = attribute(input_data, **kwargs[input_kwargs])
outputs.update(attribute_output)
return BatchFeature(outputs)
image_special_ids = self.image_processor.mm_encoder.special_ids
self.image_break_id = image_special_ids.img_break
self.image_token_id = image_special_ids.img
self.image_end_id = image_special_ids.img_end
+8 -54
View File
@@ -8,9 +8,6 @@ import torch
from mistral_common.tokens.tokenizers.audio import AudioEncoder
from transformers import BatchFeature, ProcessorMixin, TensorType
from transformers.audio_utils import AudioInput
from transformers.image_utils import ImageInput
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
from transformers.video_utils import VideoInput
from vllm.tokenizers.mistral import MistralTokenizer
@@ -62,58 +59,15 @@ class MistralCommonVoxtralProcessor(ProcessorMixin):
def __init__(self, tokenizer: MistralTokenizer) -> None:
self.tokenizer = tokenizer.transformers_tokenizer
# Back-compatibility for Transformers v4
if not hasattr(self.tokenizer, "init_kwargs"):
self.tokenizer.init_kwargs = {}
self.feature_extractor = MistralCommonFeatureExtractor(
tokenizer.instruct.audio_encoder
)
self._audio_special_ids = self.feature_extractor.audio_encoder.special_ids
@property
def audio_token_id(self) -> int:
return self._audio_special_ids.audio
@property
def begin_audio_token_id(self) -> int:
return self._audio_special_ids.begin_audio
def __call__(
self,
images: ImageInput | None = None,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput]
| None = None,
videos: VideoInput | None = None,
audio: AudioInput | None = None,
**kwargs,
):
if images is None and text is None and videos is None and audio is None:
raise ValueError(
f"You need to provide at least one input to "
f"call {self.__class__.__name__}"
)
kwargs = self._merge_kwargs(
self.valid_processor_kwargs,
tokenizer_init_kwargs={},
**kwargs,
)
kwargs["text_kwargs"]["return_tensors"] = "pt"
kwargs["audio_kwargs"]["return_tensors"] = None # Avoid padding issue
attribute_to_kwargs = {
"tokenizer": (text, "text_kwargs"),
"image_processor": (images, "images_kwargs"),
"video_processor": (videos, "videos_kwargs"),
"feature_extractor": (audio, "audio_kwargs"),
}
outputs = {}
for attribute_name in self.attributes:
attribute = getattr(self, attribute_name, None)
input_data, input_kwargs = attribute_to_kwargs[attribute_name]
if input_data is not None and attribute is not None:
attribute_output = attribute(input_data, **kwargs[input_kwargs])
outputs.update(attribute_output)
return BatchFeature(outputs)
audio_special_ids = self.feature_extractor.audio_encoder.special_ids
self.audio_token_id = audio_special_ids.audio
self.begin_audio_token_id = audio_special_ids.begin_audio