[Misc] Move processors to transformers_utils (#35953)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -15,10 +15,14 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",
|
||||
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
|
||||
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
|
||||
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
|
||||
"HunYuanVLProcessor": "vllm.transformers_utils.processors.hunyuan_vl",
|
||||
"HunYuanVLImageProcessor": "vllm.transformers_utils.processors.hunyuan_vl_image",
|
||||
"MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral",
|
||||
"MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral",
|
||||
"OvisProcessor": "vllm.transformers_utils.processors.ovis",
|
||||
"Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
|
||||
"QwenVLProcessor": "vllm.transformers_utils.processors.qwen_vl",
|
||||
"Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr",
|
||||
}
|
||||
|
||||
@@ -28,10 +32,14 @@ __all__ = [
|
||||
"DeepseekVLV2Processor",
|
||||
"FireRedASR2Processor",
|
||||
"FunASRProcessor",
|
||||
"GLM4VProcessor",
|
||||
"HunYuanVLProcessor",
|
||||
"HunYuanVLImageProcessor",
|
||||
"MistralCommonPixtralProcessor",
|
||||
"MistralCommonVoxtralProcessor",
|
||||
"OvisProcessor",
|
||||
"Ovis2_5Processor",
|
||||
"QwenVLProcessor",
|
||||
"Qwen3ASRProcessor",
|
||||
]
|
||||
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from transformers import PreTrainedTokenizer
|
||||
from transformers.image_processing_utils_fast import BaseImageProcessorFast
|
||||
from transformers.image_utils import PILImageResampling
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
|
||||
class GLM4VImageProcessorFast(BaseImageProcessorFast):
|
||||
"""
|
||||
Port of https://huggingface.co/zai-org/glm-4v-9b/blob/main/tokenization_chatglm.py#L177
|
||||
to HF Transformers.
|
||||
"""
|
||||
|
||||
resample = PILImageResampling.BICUBIC
|
||||
image_mean = [0.48145466, 0.4578275, 0.40821073]
|
||||
image_std = [0.26862954, 0.26130258, 0.27577711]
|
||||
size = {"height": 1120, "width": 1120}
|
||||
do_resize = True
|
||||
do_rescale = True
|
||||
do_normalize = True
|
||||
|
||||
|
||||
class GLM4VProcessor(ProcessorMixin):
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
image_size: int,
|
||||
) -> None:
|
||||
self.tokenizer = tokenizer
|
||||
self.image_processor = GLM4VImageProcessorFast(
|
||||
size={"width": image_size, "height": image_size}
|
||||
)
|
||||
@@ -0,0 +1,116 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from mistral_common.protocol.instruct.chunk import ImageChunk
|
||||
from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
|
||||
from PIL import Image
|
||||
from transformers import BatchFeature, ProcessorMixin, TensorType
|
||||
from transformers.audio_utils import AudioInput
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
||||
from transformers.video_utils import VideoInput
|
||||
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
|
||||
class MistralCommonImageProcessor:
|
||||
"""
|
||||
Provide a HF-compatible interface for
|
||||
`mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
|
||||
"""
|
||||
|
||||
def __init__(self, mm_encoder: ImageEncoder) -> None:
|
||||
self.mm_encoder = mm_encoder
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: ImageInput,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
images_lst = [images] if not isinstance(images, list) else images
|
||||
|
||||
images_processed = list[torch.Tensor]()
|
||||
|
||||
for image in images_lst:
|
||||
image_inputs = self.mm_encoder(ImageChunk(image=image))
|
||||
image_processed = torch.tensor(image_inputs.image)
|
||||
|
||||
images_processed.append(image_processed)
|
||||
|
||||
return BatchFeature({"images": images_processed}, tensor_type=return_tensors)
|
||||
|
||||
def get_number_of_image_patches(
|
||||
self,
|
||||
height: int,
|
||||
width: int,
|
||||
) -> tuple[int, int, int]:
|
||||
image = Image.new("RGB", (width, height))
|
||||
ncols, nrows = self.mm_encoder._image_to_num_tokens(image)
|
||||
return ncols * nrows, nrows, ncols
|
||||
|
||||
|
||||
class MistralCommonPixtralProcessor(ProcessorMixin):
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
|
||||
def __init__(self, tokenizer: MistralTokenizer) -> None:
|
||||
self.tokenizer = tokenizer.transformers_tokenizer
|
||||
self.image_processor = MistralCommonImageProcessor(
|
||||
tokenizer.instruct.mm_encoder
|
||||
)
|
||||
|
||||
self._image_special_ids = self.image_processor.mm_encoder.special_ids
|
||||
|
||||
@property
|
||||
def image_break_id(self) -> int:
|
||||
return self._image_special_ids.img_break
|
||||
|
||||
@property
|
||||
def image_token_id(self) -> int:
|
||||
return self._image_special_ids.img
|
||||
|
||||
@property
|
||||
def image_end_id(self) -> int:
|
||||
return self._image_special_ids.img_end
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: ImageInput | None = None,
|
||||
text: TextInput
|
||||
| PreTokenizedInput
|
||||
| list[TextInput]
|
||||
| list[PreTokenizedInput]
|
||||
| None = None,
|
||||
videos: VideoInput | None = None,
|
||||
audio: AudioInput | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
if images is None and text is None and videos is None and audio is None:
|
||||
raise ValueError(
|
||||
f"You need to provide at least one input to "
|
||||
f"call {self.__class__.__name__}"
|
||||
)
|
||||
|
||||
kwargs = self._merge_kwargs(
|
||||
self.valid_processor_kwargs,
|
||||
tokenizer_init_kwargs={},
|
||||
**kwargs,
|
||||
)
|
||||
kwargs["text_kwargs"]["return_tensors"] = "pt"
|
||||
kwargs["images_kwargs"]["return_tensors"] = None # Avoid padding issue
|
||||
|
||||
attribute_to_kwargs = {
|
||||
"tokenizer": (text, "text_kwargs"),
|
||||
"image_processor": (images, "images_kwargs"),
|
||||
"video_processor": (videos, "videos_kwargs"),
|
||||
"feature_extractor": (audio, "audio_kwargs"),
|
||||
}
|
||||
outputs = {}
|
||||
for attribute_name in self.attributes:
|
||||
attribute = getattr(self, attribute_name, None)
|
||||
input_data, input_kwargs = attribute_to_kwargs[attribute_name]
|
||||
if input_data is not None and attribute is not None:
|
||||
attribute_output = attribute(input_data, **kwargs[input_kwargs])
|
||||
outputs.update(attribute_output)
|
||||
|
||||
return BatchFeature(outputs)
|
||||
@@ -0,0 +1,48 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from transformers.image_processing_utils_fast import BaseImageProcessorFast
|
||||
from transformers.image_utils import PILImageResampling
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from vllm.tokenizers.qwen_vl import QwenVLTokenizer
|
||||
|
||||
|
||||
class QwenVLImageProcessorFast(BaseImageProcessorFast):
|
||||
"""
|
||||
Port of https://huggingface.co/Qwen/Qwen-VL/blob/main/visual.py#L354
|
||||
to HF Transformers.
|
||||
"""
|
||||
|
||||
resample = PILImageResampling.BICUBIC
|
||||
image_mean = [0.48145466, 0.4578275, 0.40821073]
|
||||
image_std = [0.26862954, 0.26130258, 0.27577711]
|
||||
size = {"height": 448, "width": 448}
|
||||
do_resize = True
|
||||
do_rescale = True
|
||||
do_normalize = True
|
||||
|
||||
|
||||
class QwenVLProcessor(ProcessorMixin):
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: QwenVLTokenizer,
|
||||
image_size: int,
|
||||
) -> None:
|
||||
self.tokenizer = tokenizer
|
||||
self.image_processor = QwenVLImageProcessorFast(
|
||||
size={"width": image_size, "height": image_size}
|
||||
)
|
||||
|
||||
@property
|
||||
def image_start_tag(self) -> str:
|
||||
return self.tokenizer.image_start_tag # type: ignore[attr-defined]
|
||||
|
||||
@property
|
||||
def image_end_tag(self) -> str:
|
||||
return self.tokenizer.image_end_tag # type: ignore[attr-defined]
|
||||
|
||||
@property
|
||||
def image_pad_tag(self) -> str:
|
||||
return self.tokenizer.image_pad_tag # type: ignore[attr-defined]
|
||||
@@ -0,0 +1,119 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from math import ceil
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from mistral_common.tokens.tokenizers.audio import AudioEncoder
|
||||
from transformers import BatchFeature, ProcessorMixin, TensorType
|
||||
from transformers.audio_utils import AudioInput
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
||||
from transformers.video_utils import VideoInput
|
||||
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
|
||||
class MistralCommonFeatureExtractor:
|
||||
"""
|
||||
Provide a HF-compatible interface for
|
||||
`mistral_common.tokens.tokenizers.multimodal.AudioEncoder`.
|
||||
"""
|
||||
|
||||
def __init__(self, audio_encoder: AudioEncoder) -> None:
|
||||
self.audio_encoder = audio_encoder
|
||||
|
||||
@property
|
||||
def sampling_rate(self):
|
||||
return self.audio_encoder.audio_config.sampling_rate
|
||||
|
||||
@property
|
||||
def frame_rate(self):
|
||||
return self.audio_encoder.audio_config.frame_rate
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
audios: AudioInput,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
audios_lst = [audios] if not isinstance(audios, list) else audios
|
||||
|
||||
audios_processed = list[torch.Tensor]()
|
||||
|
||||
for audio in audios_lst:
|
||||
audio = np.asarray(audio, dtype=np.float32).ravel()
|
||||
if not self.audio_encoder.audio_config.is_streaming:
|
||||
audio = self.audio_encoder.pad(audio, self.sampling_rate)
|
||||
|
||||
audios_processed.append(torch.tensor(audio))
|
||||
|
||||
return BatchFeature(
|
||||
{"audio_arrays": audios_processed}, tensor_type=return_tensors
|
||||
)
|
||||
|
||||
def get_num_audio_tokens(self, audio_length: int) -> int:
|
||||
return ceil(audio_length / (self.sampling_rate // self.frame_rate))
|
||||
|
||||
|
||||
class MistralCommonVoxtralProcessor(ProcessorMixin):
|
||||
attributes = ["feature_extractor", "tokenizer"]
|
||||
|
||||
def __init__(self, tokenizer: MistralTokenizer) -> None:
|
||||
self.tokenizer = tokenizer.transformers_tokenizer
|
||||
self.feature_extractor = MistralCommonFeatureExtractor(
|
||||
tokenizer.instruct.audio_encoder
|
||||
)
|
||||
|
||||
self._audio_special_ids = self.feature_extractor.audio_encoder.special_ids
|
||||
|
||||
@property
|
||||
def audio_token_id(self) -> int:
|
||||
return self._audio_special_ids.audio
|
||||
|
||||
@property
|
||||
def begin_audio_token_id(self) -> int:
|
||||
return self._audio_special_ids.begin_audio
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: ImageInput | None = None,
|
||||
text: TextInput
|
||||
| PreTokenizedInput
|
||||
| list[TextInput]
|
||||
| list[PreTokenizedInput]
|
||||
| None = None,
|
||||
videos: VideoInput | None = None,
|
||||
audio: AudioInput | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
if images is None and text is None and videos is None and audio is None:
|
||||
raise ValueError(
|
||||
f"You need to provide at least one input to "
|
||||
f"call {self.__class__.__name__}"
|
||||
)
|
||||
|
||||
kwargs = self._merge_kwargs(
|
||||
self.valid_processor_kwargs,
|
||||
tokenizer_init_kwargs={},
|
||||
**kwargs,
|
||||
)
|
||||
kwargs["text_kwargs"]["return_tensors"] = "pt"
|
||||
kwargs["audio_kwargs"]["return_tensors"] = None # Avoid padding issue
|
||||
|
||||
attribute_to_kwargs = {
|
||||
"tokenizer": (text, "text_kwargs"),
|
||||
"image_processor": (images, "images_kwargs"),
|
||||
"video_processor": (videos, "videos_kwargs"),
|
||||
"feature_extractor": (audio, "audio_kwargs"),
|
||||
}
|
||||
outputs = {}
|
||||
for attribute_name in self.attributes:
|
||||
attribute = getattr(self, attribute_name, None)
|
||||
input_data, input_kwargs = attribute_to_kwargs[attribute_name]
|
||||
if input_data is not None and attribute is not None:
|
||||
attribute_output = attribute(input_data, **kwargs[input_kwargs])
|
||||
outputs.update(attribute_output)
|
||||
|
||||
return BatchFeature(outputs)
|
||||
Reference in New Issue
Block a user