[Model] Use helper function to run MM processors with token inputs (where applicable) (#38018)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,16 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
from typing import Any, TypedDict
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from PIL import Image
|
||||
from transformers import BatchFeature, ProcessorMixin, TensorType
|
||||
from typing_extensions import TypedDict, Unpack
|
||||
from transformers.processing_utils import ProcessingKwargs
|
||||
from typing_extensions import Unpack
|
||||
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
@@ -308,15 +307,22 @@ def process_vision_for_patches(
|
||||
return patches, dims_virtual
|
||||
|
||||
|
||||
class IsaacImageProcessorKwargs(TypedDict, total=False):
|
||||
class IsaacImagesKwargs(TypedDict, total=False):
|
||||
patch_size: int
|
||||
max_num_patches: int
|
||||
min_num_patches: int
|
||||
pixel_shuffle_scale: int
|
||||
|
||||
|
||||
class IsaacProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
|
||||
images_kwargs: IsaacImagesKwargs
|
||||
_defaults = {
|
||||
"text_kwargs": {"padding": False},
|
||||
"images_kwargs": {},
|
||||
}
|
||||
|
||||
|
||||
class IsaacImageProcessor:
|
||||
valid_kwargs = IsaacImageProcessorKwargs
|
||||
model_input_names = ["pixel_values", "image_grid_thw"]
|
||||
|
||||
def __init__(
|
||||
@@ -335,7 +341,7 @@ class IsaacImageProcessor:
|
||||
self,
|
||||
images: Image.Image | list[Image.Image],
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs: Unpack[IsaacImageProcessorKwargs],
|
||||
**kwargs: Unpack[IsaacImagesKwargs],
|
||||
) -> BatchFeature:
|
||||
"""Preprocess images into format compatible with vLLM input processing."""
|
||||
if not isinstance(images, list):
|
||||
@@ -349,10 +355,16 @@ class IsaacImageProcessor:
|
||||
|
||||
patches, dims_virtual = process_vision_for_patches(
|
||||
image_tensor,
|
||||
patch_size=self.patch_size,
|
||||
max_num_patches=self.vision_max_num_patches,
|
||||
min_num_patches=self.vision_min_num_patches,
|
||||
pixel_shuffle_scale=self.pixel_shuffle_scale,
|
||||
patch_size=kwargs.get("patch_size", self.patch_size),
|
||||
max_num_patches=kwargs.get(
|
||||
"max_num_patches", self.vision_max_num_patches
|
||||
),
|
||||
min_num_patches=kwargs.get(
|
||||
"min_num_patches", self.vision_min_num_patches
|
||||
),
|
||||
pixel_shuffle_scale=kwargs.get(
|
||||
"pixel_shuffle_scale", self.pixel_shuffle_scale
|
||||
),
|
||||
)
|
||||
|
||||
# Isaac packs a dummy temporal dim for images
|
||||
@@ -405,13 +417,17 @@ class IsaacProcessor(ProcessorMixin):
|
||||
text: str | list[str] | None = None,
|
||||
images: Image.Image | list[Image.Image] | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
**kwargs: Unpack[IsaacProcessorKwargs], # type: ignore[misc]
|
||||
) -> BatchFeature:
|
||||
output_kwargs = self._merge_kwargs(
|
||||
IsaacProcessorKwargs,
|
||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if images is not None:
|
||||
image_inputs = self.image_processor(
|
||||
images,
|
||||
return_tensors=return_tensors,
|
||||
**kwargs,
|
||||
images, **output_kwargs["images_kwargs"]
|
||||
)
|
||||
image_grid_thw = image_inputs["image_grid_thw"]
|
||||
else:
|
||||
@@ -435,7 +451,7 @@ class IsaacProcessor(ProcessorMixin):
|
||||
index += 1
|
||||
text[i] = text[i].replace("<|placeholder|>", "<|image_pad|>")
|
||||
|
||||
text_inputs = self.tokenizer(text, return_tensors=return_tensors)
|
||||
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
|
||||
else:
|
||||
text_inputs = {}
|
||||
|
||||
|
||||
@@ -5,10 +5,7 @@ from mistral_common.protocol.instruct.chunk import ImageChunk
|
||||
from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
|
||||
from PIL import Image
|
||||
from transformers import BatchFeature, ProcessorMixin, TensorType
|
||||
from transformers.audio_utils import AudioInput
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
||||
from transformers.video_utils import VideoInput
|
||||
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
@@ -55,62 +52,16 @@ class MistralCommonPixtralProcessor(ProcessorMixin):
|
||||
|
||||
def __init__(self, tokenizer: MistralTokenizer) -> None:
|
||||
self.tokenizer = tokenizer.transformers_tokenizer
|
||||
|
||||
# Back-compatibility for Transformers v4
|
||||
if not hasattr(self.tokenizer, "init_kwargs"):
|
||||
self.tokenizer.init_kwargs = {}
|
||||
|
||||
self.image_processor = MistralCommonImageProcessor(
|
||||
tokenizer.instruct.mm_encoder
|
||||
)
|
||||
|
||||
self._image_special_ids = self.image_processor.mm_encoder.special_ids
|
||||
|
||||
@property
|
||||
def image_break_id(self) -> int:
|
||||
return self._image_special_ids.img_break
|
||||
|
||||
@property
|
||||
def image_token_id(self) -> int:
|
||||
return self._image_special_ids.img
|
||||
|
||||
@property
|
||||
def image_end_id(self) -> int:
|
||||
return self._image_special_ids.img_end
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: ImageInput | None = None,
|
||||
text: TextInput
|
||||
| PreTokenizedInput
|
||||
| list[TextInput]
|
||||
| list[PreTokenizedInput]
|
||||
| None = None,
|
||||
videos: VideoInput | None = None,
|
||||
audio: AudioInput | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
if images is None and text is None and videos is None and audio is None:
|
||||
raise ValueError(
|
||||
f"You need to provide at least one input to "
|
||||
f"call {self.__class__.__name__}"
|
||||
)
|
||||
|
||||
kwargs = self._merge_kwargs(
|
||||
self.valid_processor_kwargs,
|
||||
tokenizer_init_kwargs={},
|
||||
**kwargs,
|
||||
)
|
||||
kwargs["text_kwargs"]["return_tensors"] = "pt"
|
||||
kwargs["images_kwargs"]["return_tensors"] = None # Avoid padding issue
|
||||
|
||||
attribute_to_kwargs = {
|
||||
"tokenizer": (text, "text_kwargs"),
|
||||
"image_processor": (images, "images_kwargs"),
|
||||
"video_processor": (videos, "videos_kwargs"),
|
||||
"feature_extractor": (audio, "audio_kwargs"),
|
||||
}
|
||||
outputs = {}
|
||||
for attribute_name in self.attributes:
|
||||
attribute = getattr(self, attribute_name, None)
|
||||
input_data, input_kwargs = attribute_to_kwargs[attribute_name]
|
||||
if input_data is not None and attribute is not None:
|
||||
attribute_output = attribute(input_data, **kwargs[input_kwargs])
|
||||
outputs.update(attribute_output)
|
||||
|
||||
return BatchFeature(outputs)
|
||||
image_special_ids = self.image_processor.mm_encoder.special_ids
|
||||
self.image_break_id = image_special_ids.img_break
|
||||
self.image_token_id = image_special_ids.img
|
||||
self.image_end_id = image_special_ids.img_end
|
||||
|
||||
@@ -8,9 +8,6 @@ import torch
|
||||
from mistral_common.tokens.tokenizers.audio import AudioEncoder
|
||||
from transformers import BatchFeature, ProcessorMixin, TensorType
|
||||
from transformers.audio_utils import AudioInput
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
||||
from transformers.video_utils import VideoInput
|
||||
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
@@ -62,58 +59,15 @@ class MistralCommonVoxtralProcessor(ProcessorMixin):
|
||||
|
||||
def __init__(self, tokenizer: MistralTokenizer) -> None:
|
||||
self.tokenizer = tokenizer.transformers_tokenizer
|
||||
|
||||
# Back-compatibility for Transformers v4
|
||||
if not hasattr(self.tokenizer, "init_kwargs"):
|
||||
self.tokenizer.init_kwargs = {}
|
||||
|
||||
self.feature_extractor = MistralCommonFeatureExtractor(
|
||||
tokenizer.instruct.audio_encoder
|
||||
)
|
||||
|
||||
self._audio_special_ids = self.feature_extractor.audio_encoder.special_ids
|
||||
|
||||
@property
|
||||
def audio_token_id(self) -> int:
|
||||
return self._audio_special_ids.audio
|
||||
|
||||
@property
|
||||
def begin_audio_token_id(self) -> int:
|
||||
return self._audio_special_ids.begin_audio
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: ImageInput | None = None,
|
||||
text: TextInput
|
||||
| PreTokenizedInput
|
||||
| list[TextInput]
|
||||
| list[PreTokenizedInput]
|
||||
| None = None,
|
||||
videos: VideoInput | None = None,
|
||||
audio: AudioInput | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
if images is None and text is None and videos is None and audio is None:
|
||||
raise ValueError(
|
||||
f"You need to provide at least one input to "
|
||||
f"call {self.__class__.__name__}"
|
||||
)
|
||||
|
||||
kwargs = self._merge_kwargs(
|
||||
self.valid_processor_kwargs,
|
||||
tokenizer_init_kwargs={},
|
||||
**kwargs,
|
||||
)
|
||||
kwargs["text_kwargs"]["return_tensors"] = "pt"
|
||||
kwargs["audio_kwargs"]["return_tensors"] = None # Avoid padding issue
|
||||
|
||||
attribute_to_kwargs = {
|
||||
"tokenizer": (text, "text_kwargs"),
|
||||
"image_processor": (images, "images_kwargs"),
|
||||
"video_processor": (videos, "videos_kwargs"),
|
||||
"feature_extractor": (audio, "audio_kwargs"),
|
||||
}
|
||||
outputs = {}
|
||||
for attribute_name in self.attributes:
|
||||
attribute = getattr(self, attribute_name, None)
|
||||
input_data, input_kwargs = attribute_to_kwargs[attribute_name]
|
||||
if input_data is not None and attribute is not None:
|
||||
attribute_output = attribute(input_data, **kwargs[input_kwargs])
|
||||
outputs.update(attribute_output)
|
||||
|
||||
return BatchFeature(outputs)
|
||||
audio_special_ids = self.feature_extractor.audio_encoder.special_ids
|
||||
self.audio_token_id = audio_special_ids.audio
|
||||
self.begin_audio_token_id = audio_special_ids.begin_audio
|
||||
|
||||
Reference in New Issue
Block a user