[Misc] Move processors to transformers_utils (#35953)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-03-09 11:31:39 +08:00
committed by GitHub
parent bd2659a566
commit d62856b928
13 changed files with 507 additions and 595 deletions
@@ -15,10 +15,14 @@ _CLASS_TO_MODULE: dict[str, str] = {
"DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
"HunYuanVLProcessor": "vllm.transformers_utils.processors.hunyuan_vl",
"HunYuanVLImageProcessor": "vllm.transformers_utils.processors.hunyuan_vl_image",
"MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral",
"MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral",
"OvisProcessor": "vllm.transformers_utils.processors.ovis",
"Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
"QwenVLProcessor": "vllm.transformers_utils.processors.qwen_vl",
"Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr",
}
@@ -28,10 +32,14 @@ __all__ = [
"DeepseekVLV2Processor",
"FireRedASR2Processor",
"FunASRProcessor",
"GLM4VProcessor",
"HunYuanVLProcessor",
"HunYuanVLImageProcessor",
"MistralCommonPixtralProcessor",
"MistralCommonVoxtralProcessor",
"OvisProcessor",
"Ovis2_5Processor",
"QwenVLProcessor",
"Qwen3ASRProcessor",
]
@@ -0,0 +1,35 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers import PreTrainedTokenizer
from transformers.image_processing_utils_fast import BaseImageProcessorFast
from transformers.image_utils import PILImageResampling
from transformers.processing_utils import ProcessorMixin
class GLM4VImageProcessorFast(BaseImageProcessorFast):
"""
Port of https://huggingface.co/zai-org/glm-4v-9b/blob/main/tokenization_chatglm.py#L177
to HF Transformers.
"""
resample = PILImageResampling.BICUBIC
image_mean = [0.48145466, 0.4578275, 0.40821073]
image_std = [0.26862954, 0.26130258, 0.27577711]
size = {"height": 1120, "width": 1120}
do_resize = True
do_rescale = True
do_normalize = True
class GLM4VProcessor(ProcessorMixin):
attributes = ["image_processor", "tokenizer"]
def __init__(
self,
tokenizer: PreTrainedTokenizer,
image_size: int,
) -> None:
self.tokenizer = tokenizer
self.image_processor = GLM4VImageProcessorFast(
size={"width": image_size, "height": image_size}
)
@@ -0,0 +1,116 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from mistral_common.protocol.instruct.chunk import ImageChunk
from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
from PIL import Image
from transformers import BatchFeature, ProcessorMixin, TensorType
from transformers.audio_utils import AudioInput
from transformers.image_utils import ImageInput
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
from transformers.video_utils import VideoInput
from vllm.tokenizers.mistral import MistralTokenizer
class MistralCommonImageProcessor:
"""
Provide a HF-compatible interface for
`mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
"""
def __init__(self, mm_encoder: ImageEncoder) -> None:
self.mm_encoder = mm_encoder
def __call__(
self,
images: ImageInput,
return_tensors: str | TensorType | None = None,
**kwargs,
) -> BatchFeature:
images_lst = [images] if not isinstance(images, list) else images
images_processed = list[torch.Tensor]()
for image in images_lst:
image_inputs = self.mm_encoder(ImageChunk(image=image))
image_processed = torch.tensor(image_inputs.image)
images_processed.append(image_processed)
return BatchFeature({"images": images_processed}, tensor_type=return_tensors)
def get_number_of_image_patches(
self,
height: int,
width: int,
) -> tuple[int, int, int]:
image = Image.new("RGB", (width, height))
ncols, nrows = self.mm_encoder._image_to_num_tokens(image)
return ncols * nrows, nrows, ncols
class MistralCommonPixtralProcessor(ProcessorMixin):
attributes = ["image_processor", "tokenizer"]
def __init__(self, tokenizer: MistralTokenizer) -> None:
self.tokenizer = tokenizer.transformers_tokenizer
self.image_processor = MistralCommonImageProcessor(
tokenizer.instruct.mm_encoder
)
self._image_special_ids = self.image_processor.mm_encoder.special_ids
@property
def image_break_id(self) -> int:
return self._image_special_ids.img_break
@property
def image_token_id(self) -> int:
return self._image_special_ids.img
@property
def image_end_id(self) -> int:
return self._image_special_ids.img_end
def __call__(
self,
images: ImageInput | None = None,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput]
| None = None,
videos: VideoInput | None = None,
audio: AudioInput | None = None,
**kwargs,
):
if images is None and text is None and videos is None and audio is None:
raise ValueError(
f"You need to provide at least one input to "
f"call {self.__class__.__name__}"
)
kwargs = self._merge_kwargs(
self.valid_processor_kwargs,
tokenizer_init_kwargs={},
**kwargs,
)
kwargs["text_kwargs"]["return_tensors"] = "pt"
kwargs["images_kwargs"]["return_tensors"] = None # Avoid padding issue
attribute_to_kwargs = {
"tokenizer": (text, "text_kwargs"),
"image_processor": (images, "images_kwargs"),
"video_processor": (videos, "videos_kwargs"),
"feature_extractor": (audio, "audio_kwargs"),
}
outputs = {}
for attribute_name in self.attributes:
attribute = getattr(self, attribute_name, None)
input_data, input_kwargs = attribute_to_kwargs[attribute_name]
if input_data is not None and attribute is not None:
attribute_output = attribute(input_data, **kwargs[input_kwargs])
outputs.update(attribute_output)
return BatchFeature(outputs)
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers.image_processing_utils_fast import BaseImageProcessorFast
from transformers.image_utils import PILImageResampling
from transformers.processing_utils import ProcessorMixin
from vllm.tokenizers.qwen_vl import QwenVLTokenizer
class QwenVLImageProcessorFast(BaseImageProcessorFast):
"""
Port of https://huggingface.co/Qwen/Qwen-VL/blob/main/visual.py#L354
to HF Transformers.
"""
resample = PILImageResampling.BICUBIC
image_mean = [0.48145466, 0.4578275, 0.40821073]
image_std = [0.26862954, 0.26130258, 0.27577711]
size = {"height": 448, "width": 448}
do_resize = True
do_rescale = True
do_normalize = True
class QwenVLProcessor(ProcessorMixin):
attributes = ["image_processor", "tokenizer"]
def __init__(
self,
tokenizer: QwenVLTokenizer,
image_size: int,
) -> None:
self.tokenizer = tokenizer
self.image_processor = QwenVLImageProcessorFast(
size={"width": image_size, "height": image_size}
)
@property
def image_start_tag(self) -> str:
return self.tokenizer.image_start_tag # type: ignore[attr-defined]
@property
def image_end_tag(self) -> str:
return self.tokenizer.image_end_tag # type: ignore[attr-defined]
@property
def image_pad_tag(self) -> str:
return self.tokenizer.image_pad_tag # type: ignore[attr-defined]
@@ -0,0 +1,119 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from math import ceil
import numpy as np
import torch
from mistral_common.tokens.tokenizers.audio import AudioEncoder
from transformers import BatchFeature, ProcessorMixin, TensorType
from transformers.audio_utils import AudioInput
from transformers.image_utils import ImageInput
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
from transformers.video_utils import VideoInput
from vllm.tokenizers.mistral import MistralTokenizer
class MistralCommonFeatureExtractor:
"""
Provide a HF-compatible interface for
`mistral_common.tokens.tokenizers.multimodal.AudioEncoder`.
"""
def __init__(self, audio_encoder: AudioEncoder) -> None:
self.audio_encoder = audio_encoder
@property
def sampling_rate(self):
return self.audio_encoder.audio_config.sampling_rate
@property
def frame_rate(self):
return self.audio_encoder.audio_config.frame_rate
def __call__(
self,
audios: AudioInput,
return_tensors: str | TensorType | None = None,
**kwargs,
) -> BatchFeature:
audios_lst = [audios] if not isinstance(audios, list) else audios
audios_processed = list[torch.Tensor]()
for audio in audios_lst:
audio = np.asarray(audio, dtype=np.float32).ravel()
if not self.audio_encoder.audio_config.is_streaming:
audio = self.audio_encoder.pad(audio, self.sampling_rate)
audios_processed.append(torch.tensor(audio))
return BatchFeature(
{"audio_arrays": audios_processed}, tensor_type=return_tensors
)
def get_num_audio_tokens(self, audio_length: int) -> int:
return ceil(audio_length / (self.sampling_rate // self.frame_rate))
class MistralCommonVoxtralProcessor(ProcessorMixin):
attributes = ["feature_extractor", "tokenizer"]
def __init__(self, tokenizer: MistralTokenizer) -> None:
self.tokenizer = tokenizer.transformers_tokenizer
self.feature_extractor = MistralCommonFeatureExtractor(
tokenizer.instruct.audio_encoder
)
self._audio_special_ids = self.feature_extractor.audio_encoder.special_ids
@property
def audio_token_id(self) -> int:
return self._audio_special_ids.audio
@property
def begin_audio_token_id(self) -> int:
return self._audio_special_ids.begin_audio
def __call__(
self,
images: ImageInput | None = None,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput]
| None = None,
videos: VideoInput | None = None,
audio: AudioInput | None = None,
**kwargs,
):
if images is None and text is None and videos is None and audio is None:
raise ValueError(
f"You need to provide at least one input to "
f"call {self.__class__.__name__}"
)
kwargs = self._merge_kwargs(
self.valid_processor_kwargs,
tokenizer_init_kwargs={},
**kwargs,
)
kwargs["text_kwargs"]["return_tensors"] = "pt"
kwargs["audio_kwargs"]["return_tensors"] = None # Avoid padding issue
attribute_to_kwargs = {
"tokenizer": (text, "text_kwargs"),
"image_processor": (images, "images_kwargs"),
"video_processor": (videos, "videos_kwargs"),
"feature_extractor": (audio, "audio_kwargs"),
}
outputs = {}
for attribute_name in self.attributes:
attribute = getattr(self, attribute_name, None)
input_data, input_kwargs = attribute_to_kwargs[attribute_name]
if input_data is not None and attribute is not None:
attribute_output = attribute(input_data, **kwargs[input_kwargs])
outputs.update(attribute_output)
return BatchFeature(outputs)