From 51e5372f3d8b2eb8cf96e20120af4de78b924fba Mon Sep 17 00:00:00 2001 From: manayang Date: Wed, 8 Jul 2026 15:58:23 +0800 Subject: [PATCH] [Model][HunyuanVL] Use native transformers processor and adapt to transformers 5.13 (#47872) Co-authored-by: manayang --- tests/models/registry.py | 2 +- vllm/model_executor/models/hunyuan_vision.py | 28 +- .../processors/hunyuan_vl.py | 226 --------- .../processors/hunyuan_vl_image.py | 477 ------------------ 4 files changed, 21 insertions(+), 712 deletions(-) delete mode 100644 vllm/transformers_utils/processors/hunyuan_vl.py delete mode 100644 vllm/transformers_utils/processors/hunyuan_vl_image.py diff --git a/tests/models/registry.py b/tests/models/registry.py index 85c53723ae..00085f44a6 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -936,7 +936,7 @@ _MULTIMODAL_EXAMPLE_MODELS = { "HunYuanVLForConditionalGeneration": _HfExamplesInfo( "tencent/HunyuanOCR", hf_overrides={"num_experts": 0}, - is_available_online=False, + min_transformers_version="5.13", ), "Idefics3ForConditionalGeneration": _HfExamplesInfo( "HuggingFaceM4/Idefics3-8B-Llama3", diff --git a/vllm/model_executor/models/hunyuan_vision.py b/vllm/model_executor/models/hunyuan_vision.py index 313e13c915..87520fe29e 100644 --- a/vllm/model_executor/models/hunyuan_vision.py +++ b/vllm/model_executor/models/hunyuan_vision.py @@ -31,7 +31,11 @@ from typing import Annotated, Any, Literal, TypeAlias import torch import torch.nn as nn import torch.nn.functional as F -from transformers import BatchFeature +from transformers import BatchFeature, HunYuanVLProcessor +from transformers.models.hunyuan_vl.image_processing_hunyuan_vl import ( + HunYuanVLImageProcessor, + smart_resize, +) from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions @@ -75,11 +79,6 @@ from vllm.transformers_utils.configs.hunyuan_vl import ( HunYuanVLConfig, HunYuanVLVisionConfig, ) -from vllm.transformers_utils.processors.hunyuan_vl import HunYuanVLProcessor -from vllm.transformers_utils.processors.hunyuan_vl_image import ( - HunYuanVLImageProcessor, - smart_resize, -) from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import ( @@ -574,9 +573,12 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo): self, **kwargs: object, ) -> HunYuanVLProcessor: + # transformers>=5.13 replaced `use_fast` with `backend`; pin the + # PIL backend to match the released HunyuanOCR checkpoint packing. + kwargs.pop("use_fast", None) + kwargs.setdefault("backend", "pil") return self.ctx.get_hf_processor( HunYuanVLProcessor, - use_fast=kwargs.pop("use_fast", True), **kwargs, ) @@ -724,8 +726,18 @@ class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingIn mm_kwargs: Mapping[str, object], tok_kwargs: Mapping[str, object], ) -> BatchFeature: + hf_processor = self.info.get_hf_processor(**mm_kwargs) + # HunYuanVLProcessor requires image placeholders wrapped with start/end tokens. + if mm_data.get("images") is not None and prompt: + img_tok = hf_processor.image_token + wrapped = ( + f"{hf_processor.image_start_token}{img_tok}" + f"{hf_processor.image_end_token}" + ) + if img_tok in prompt and wrapped not in prompt: + prompt = prompt.replace(img_tok, wrapped) return self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**mm_kwargs), + hf_processor, dict(text=prompt, **mm_data), dict(**mm_kwargs, **tok_kwargs), ) diff --git a/vllm/transformers_utils/processors/hunyuan_vl.py b/vllm/transformers_utils/processors/hunyuan_vl.py deleted file mode 100644 index 2d0e4db97a..0000000000 --- a/vllm/transformers_utils/processors/hunyuan_vl.py +++ /dev/null @@ -1,226 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# adapted from https://github.com/ManaEstras/transformers/blob/v4.57.1.hyvl/src/transformers/models/hunyuan_vl/processing_hunyuan_vl.py - -import numpy as np -import torch -from transformers.feature_extraction_utils import BatchFeature -from transformers.image_utils import ImageInput -from transformers.processing_utils import ProcessorMixin -from transformers.tokenization_utils_base import PreTokenizedInput, TextInput -from transformers.video_utils import VideoInput - - -class HunYuanVLProcessor(ProcessorMixin): - attributes = ["image_processor", "tokenizer"] - valid_kwargs = ["chat_template"] - image_processor_class = "AutoImageProcessor" - tokenizer_class = "AutoTokenizer" # ("AutoTokenizer", None) - - def __init__( - self, - image_processor=None, - tokenizer=None, - chat_template=None, - **kwargs, - ): - # TODO Fix the init - self.tokenizer = tokenizer - self.image_token_id = 120120 # self.tokenizer.image_token_id - self.image_token = self.tokenizer.convert_ids_to_tokens(self.image_token_id) - self.im_start_token_id = 120118 # self.tokenizer.im_start_id - self.im_start_token = self.tokenizer.convert_ids_to_tokens( - self.im_start_token_id - ) - self.im_end_token_id = 120119 # self.tokenizer.im_end_id - self.im_end_token = self.tokenizer.convert_ids_to_tokens(self.im_end_token_id) - self.placeholder_token = self.tokenizer.convert_ids_to_tokens( - self.tokenizer.vocab_size - 1 - ) - self.pad_id = 120002 # self.tokenizer.pad_token_id - - super().__init__(image_processor, tokenizer, chat_template=chat_template) - - def __call__( - self, - images: ImageInput = None, - text: TextInput - | PreTokenizedInput - | list[TextInput] - | list[PreTokenizedInput] = None, - videos: VideoInput = None, - **kwargs, - ) -> BatchFeature: - image_inputs = {} - if images is not None: - image_inputs = self.image_processor(images=images) - image_grid_thw = image_inputs["image_grid_thw"] - - if not isinstance(text, list): - text = [text] - - text = text.copy() # below lines change text in-place - - image_tokens_cumsum = [0] - if images is not None: - index = 0 - for i in range(len(text)): - while self.image_token in text[i]: - grid_h, grid_w = image_grid_thw[index][-2:] - patch_h = grid_h // self.image_processor.merge_size - patch_w = grid_w // self.image_processor.merge_size - num_image_tokens = patch_h * (patch_w + 1) + 2 - image_tokens_cumsum.append( - image_tokens_cumsum[-1] + num_image_tokens - ) - # text[i] = text[i].replace(self.image_token, self.im_start_token + self.placeholder_token * num_image_tokens + self.im_end_token, 1) # noqa: E501 - text[i] = text[i].replace( - self.image_token, self.placeholder_token * num_image_tokens, 1 - ) - index += 1 - text[i] = text[i].replace(self.placeholder_token, self.image_token) - # text[i] = self.tokenizer.bos_token + text[i] - - text_inputs = self.tokenizer(text, add_special_tokens=False, **kwargs) - self._check_special_mm_tokens(text, text_inputs, modalities=["image"]) - - input_ids = text_inputs["input_ids"] - position_ids = torch.arange(len(input_ids[0])) - position_ids_w = torch.arange(len(input_ids[0])) - position_ids_h = torch.arange(len(input_ids[0])) - position_ids_t = torch.arange(len(input_ids[0])) - - if images is not None: - image_token_pos_indices = torch.where(input_ids[0] == self.image_token_id)[ - 0 - ] - for i in range(len(image_grid_thw)): - grid_h, grid_w = image_grid_thw[i][-2:] - patch_h = grid_h // self.image_processor.merge_size - patch_w = grid_w // self.image_processor.merge_size - start_pos = image_token_pos_indices[image_tokens_cumsum[i]].item() + 1 - replace_num = (patch_w + 1) * patch_h - position_ids_w[start_pos : start_pos + replace_num] = torch.tensor( - list(range(patch_w + 1)) * patch_h, dtype=torch.int64 - ) - patch_h_list = [] - for h in range(patch_h): - patch_h_list += [h] * (patch_w + 1) - position_ids_h[start_pos : start_pos + replace_num] = torch.tensor( - patch_h_list, dtype=torch.int64 - ) - position_ids_t[start_pos : start_pos + replace_num] = 0 - - position_ids = torch.stack( - [position_ids, position_ids_w, position_ids_h, position_ids_t] - ).unsqueeze(0) - text_inputs["position_ids"] = position_ids - - attention_mask = input_ids.ne(self.pad_id) - text_inputs["attention_mask"] = attention_mask - text_inputs["imgs_pos"] = [self.get_imgs_pos(e) for e in input_ids] - # image_inputs["imgs"] = [[image_inputs["pixel_values"]]] - - return_tensors = kwargs.pop("return_tensors", None) - return BatchFeature( - data={**text_inputs, **image_inputs}, - tensor_type=return_tensors, - ) - - def batch_decode(self, *args, **kwargs): - return self.tokenizer.batch_decode(*args, **kwargs) - - def decode(self, *args, **kwargs): - return self.tokenizer.decode(*args, **kwargs) - - def post_process_image_text_to_text( - self, - generated_outputs, - skip_special_tokens=True, - clean_up_tokenization_spaces=False, - **kwargs, - ): - assert 0 - - def apply_chat_template(self, *args, **kwargs): - kwargs["return_dict"] = False - return self.tokenizer.apply_chat_template(*args, **kwargs) - - def get_imgs_pos(self, doc_ids): - doc_ids = np.array(doc_ids, dtype=np.int64) - img_begin_index = np.where(doc_ids == self.im_start_token_id)[0] - img_end_index = np.where(doc_ids == self.im_end_token_id)[0] - imgs_pos = np.concatenate( - ( - np.reshape(img_begin_index + 1, (-1, 1)), - np.reshape(img_end_index, (-1, 1)), - ), - axis=-1, - ).tolist() - return imgs_pos - - @property - def model_input_names(self): - tokenizer_input_names = self.tokenizer.model_input_names - image_processor_input_names = self.image_processor.model_input_names - return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) - - -def split_image_into_patch_blocks( - pixel_values: torch.Tensor, # shape: [batch_size, 3, H, W] - patch_size: int = 16, # e.g. 16 - adaptor_patch_div: int = 4, # e.g. 4 --> each patch_size is cut into 4x4 small regions, i.e. patch_size // 4 # noqa: E501 -) -> torch.Tensor: - """ - Split the input image tensor (supporting batch) into large patches of size `patch_size`, - and then further divide each large patch into smaller regions of size - (patch_size // adaptor_patch_div) x (patch_size // adaptor_patch_div). - Each small region is extracted as a tensor of shape [3, patch_size, patch_size]. - The final output contains all such small region tensors. - - Args: - pixel_values: Input image tensor of shape [batch_size, 3, H, W]. - patch_size: Size of the large patch, e.g., 16. - adaptor_patch_div: Each large patch is divided into - (patch_size // adaptor_patch_div) x (patch_size // adaptor_patch_div) - smaller regions. - - Returns: - patches: A tensor of shape [N, 3, patch_size, patch_size], - where N = batch_size * (H // patch_size) * (W // patch_size) * (patch_size // adaptor_patch_div)^2. - Each element in the batch corresponds to one small image region. - """ # noqa: E501 - batch_size, channels, height, width = pixel_values.shape - assert channels == 3, "Pixel values must have 3 channels in dim=1" - assert height % patch_size == 0 and width % patch_size == 0, ( - "H and W must be divisible by patch_size" - ) - - patch_height_num = height // patch_size - patch_width_num = width // patch_size - - # Reshape to [B, 3, ph, ps, pw, ps] - img = pixel_values.reshape( - batch_size, 3, patch_height_num, patch_size, patch_width_num, patch_size - ) - - # Further split each psxps patch into (ps//aps)x(ps//aps) small regions - img = img.reshape( - batch_size, - 3, - patch_height_num, - patch_size // adaptor_patch_div, # ps // aps - adaptor_patch_div, - patch_width_num, - patch_size // adaptor_patch_div, # ps // aps - adaptor_patch_div, - ) - - # Permute to group the small regions: [B, ph, pw, ps//aps, ps//aps, 3, aps, aps] - img = img.permute(0, 2, 5, 3, 6, 1, 4, 7) - - # Reshape into [B * ph * pw * (ps//aps)^2, 3, patch_size, patch_size] - patches = img.reshape(-1, 3, patch_size, patch_size) - - return patches diff --git a/vllm/transformers_utils/processors/hunyuan_vl_image.py b/vllm/transformers_utils/processors/hunyuan_vl_image.py deleted file mode 100644 index 0b10ae249d..0000000000 --- a/vllm/transformers_utils/processors/hunyuan_vl_image.py +++ /dev/null @@ -1,477 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# adapted from https://github.com/ManaEstras/transformers/blob/v4.57.1.hyvl/src/transformers/models/hunyuan_vl/image_processing_hunyuan_vl.py -"""Image processor class for HunYuanVL.""" - -# isort conflicts with ruff for transformers imports -# isort: skip_file -import math - -import numpy as np -import torchvision.transforms as transforms -from transformers import AutoImageProcessor -from transformers.image_processing_utils import BaseImageProcessor, BatchFeature -from transformers.image_transforms import ( - convert_to_rgb, -) -from transformers.image_utils import ( - OPENAI_CLIP_MEAN, - OPENAI_CLIP_STD, - ChannelDimension, - ImageInput, - PILImageResampling, - make_flat_list_of_images, - make_list_of_images, - valid_images, - validate_preprocess_arguments, -) -from transformers.utils import TensorType, logging -from transformers.video_utils import VideoInput, make_batched_videos - -logger = logging.get_logger(__name__) - - -def smart_resize( - height: int, - width: int, - factor: int = 16, - min_pixels: int = 512 * 512, - max_pixels: int = 2048 * 2048, -): - """Rescales the image so that the following conditions are met: - - 1. Both dimensions (height and width) are divisible by 'factor'. - - 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. - - 3. The aspect ratio of the image is maintained as closely as possible. - - """ - if max(height, width) / min(height, width) > 200: - raise ValueError( - "absolute aspect ratio must be smaller than 200, got " - f"{max(height, width) / min(height, width)}" - ) - h_bar = round(height / factor) * factor - w_bar = round(width / factor) * factor - if h_bar * w_bar > max_pixels: - beta = math.sqrt((height * width) / max_pixels) - h_bar = max(factor, math.floor(height / beta / factor) * factor) - w_bar = max(factor, math.floor(width / beta / factor) * factor) - elif h_bar * w_bar < min_pixels: - beta = math.sqrt(min_pixels / (height * width)) - h_bar = math.ceil(height * beta / factor) * factor - w_bar = math.ceil(width * beta / factor) * factor - return h_bar, w_bar - - -class HunYuanVLImageProcessor(BaseImageProcessor): - model_input_names = [ - "pixel_values", - "image_grid_thw", - "pixel_values_videos", - "video_grid_thw", - ] - - def __init__( - self, - do_resize: bool = True, - size: dict[str, int] | None = None, - resample: PILImageResampling = PILImageResampling.BICUBIC, - do_rescale: bool = True, - rescale_factor: int | float = 1 / 255, - do_normalize: bool = True, - image_mean: float | list[float] | None = None, - image_std: float | list[float] | None = None, - do_convert_rgb: bool = True, - min_pixels: int | None = None, - max_pixels: int | None = None, - patch_size: int = 16, - temporal_patch_size: int = 2, - merge_size: int = 2, - **kwargs, - ) -> None: - super().__init__(**kwargs) - if size is not None and ( - "shortest_edge" not in size or "longest_edge" not in size - ): - raise ValueError( - "size must contain 'shortest_edge' and 'longest_edge' keys." - ) - else: - size = {"shortest_edge": 512 * 512, "longest_edge": 2048 * 2048} - # backward compatibility: override size with min_pixels and max_pixels - # if they are provided. - if min_pixels is not None: - size["shortest_edge"] = min_pixels - if max_pixels is not None: - size["longest_edge"] = max_pixels - self.min_pixels = size["shortest_edge"] - self.max_pixels = size["longest_edge"] - self.size = size - - self.do_resize = do_resize - self.resample = resample - self.do_rescale = do_rescale - self.rescale_factor = rescale_factor - self.do_normalize = do_normalize - self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN - self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD - - self.patch_size = patch_size - self.temporal_patch_size = temporal_patch_size - self.merge_size = merge_size - self.do_convert_rgb = do_convert_rgb - - # hard-code - - def _preprocess( - self, - images: ImageInput | VideoInput, - do_resize: bool | None = None, - size: dict[str, int] | None = None, - resample: PILImageResampling = None, - do_rescale: bool | None = None, - rescale_factor: float | None = None, - do_normalize: bool | None = None, - image_mean: float | list[float] | None = None, - image_std: float | list[float] | None = None, - patch_size: int = 16, - temporal_patch_size: int = 2, - merge_size: int = 2, - do_convert_rgb: bool | None = None, - data_format: ChannelDimension | None = ChannelDimension.FIRST, - input_data_format: str | ChannelDimension | None = None, - ): - """ - Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`. - - Args: - images (`ImageInput`): - Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255. If pixel values range from 0 to 1, set `do_rescale=False`. - do_resize (`bool`, *optional*, defaults to `self.do_resize`): - Whether to resize the image. - size (`dict[str, int]`, *optional*, defaults to `self.size`): - Size of the image after resizing. `shortest_edge` and `longest_edge` keys must be present. - resample (`PILImageResampling`, *optional*, defaults to `self.resample`): - Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums. - do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): - Whether to rescale the image. - rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): - Scale factor to use if rescaling the image. - do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): - Whether to normalize the image. - image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`): - Mean to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image. - image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`): - Standard deviation to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image. - patch_size (`int`, *optional*, defaults to `self.patch_size`): - The spatial patch size of the vision encoder. - temporal_patch_size (`int`, *optional*, defaults to `self.temporal_patch_size`): - The temporal patch size of the vision encoder. - merge_size (`int`, *optional*, defaults to `self.merge_size`): - The merge size of the vision encoder to llm encoder. - do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): - Whether to convert the image to RGB. - data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`): - The channel dimension format for the output image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Unset: Use the channel dimension format of the input image. - input_data_format (`ChannelDimension` or `str`, *optional*): - The channel dimension format for the input image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - """ # noqa: E501 - images = make_list_of_images(images) - - if do_convert_rgb: - images = [convert_to_rgb(image) for image in images] - - width, height = images[0].width, images[0].height - resized_width, resized_height = width, height - processed_images = [] - for image in images: - if do_resize: - resized_height, resized_width = smart_resize( - height=height, - width=width, - factor=patch_size * merge_size, - min_pixels=self.min_pixels, - max_pixels=self.max_pixels, - ) - image = image.resize((resized_width, resized_height)) - - if do_normalize: - image = transforms.Compose( - [ - transforms.ToTensor(), - transforms.Normalize(self.image_mean, self.image_std), - ] - )(image) - processed_images.append(image) - - patches = np.array(processed_images) - channel = patches.shape[1] - grid_t = patches.shape[0] // temporal_patch_size - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size - patches = patches.reshape( - 1, - channel, - grid_h // merge_size, - merge_size, - patch_size, - grid_w // merge_size, - merge_size, - patch_size, - ) - patches = patches.transpose(0, 2, 3, 5, 6, 1, 4, 7) - flatten_patches = patches.reshape( - 1 * grid_h * grid_w, channel * patch_size * patch_size - ) - - return flatten_patches, (grid_t, grid_h, grid_w) - - def preprocess( - self, - images: ImageInput, - videos: VideoInput = None, - do_resize: bool | None = None, - size: dict[str, int] | None = None, - min_pixels: int | None = None, - max_pixels: int | None = None, - resample: PILImageResampling = None, - do_rescale: bool | None = None, - rescale_factor: float | None = None, - do_normalize: bool | None = None, - image_mean: float | list[float] | None = None, - image_std: float | list[float] | None = None, - patch_size: int | None = None, - temporal_patch_size: int | None = None, - merge_size: int | None = None, - do_convert_rgb: bool | None = None, - return_tensors: str | TensorType | None = None, - data_format: ChannelDimension | None = ChannelDimension.FIRST, - input_data_format: str | ChannelDimension | None = None, - ): - """ - Args: - images (`ImageInput`): - Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If - passing in images with pixel values between 0 and 1, set `do_rescale=False`. - videos (`VideoInput`): - Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If - passing in videos with pixel values between 0 and 1, set `do_rescale=False`. - do_resize (`bool`, *optional*, defaults to `self.do_resize`): - Whether to resize the image. - size (`dict[str, int]`, *optional*, defaults to `self.size`): - Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with - the longest edge resized to keep the input aspect ratio. - resample (`int`, *optional*, defaults to `self.resample`): - Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only - has an effect if `do_resize` is set to `True`. - do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): - Whether to rescale the image. - rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): - Rescale factor to rescale the image by if `do_rescale` is set to `True`. - do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): - Whether to normalize the image. - image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`): - Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`. - image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`): - Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to - `True`. - min_pixels (`int`, *optional*, defaults to `self.min_pixels`): - The min pixels of the image to resize the image. - max_pixels (`int`, *optional*, defaults to `self.max_pixels`): - The max pixels of the image to resize the image. - patch_size (`int`, *optional*, defaults to `self.patch_size`): - The spatial patch size of the vision encoder. - temporal_patch_size (`int`, *optional*, defaults to `self.temporal_patch_size`): - The temporal patch size of the vision encoder. - merge_size (`int`, *optional*, defaults to `self.merge_size`): - The merge size of the vision encoder to llm encoder. - do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): - Whether to convert the image to RGB. - return_tensors (`str` or `TensorType`, *optional*): - The type of tensors to return. Can be one of: - - Unset: Return a list of `np.ndarray`. - - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. - - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. - - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. - data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): - The channel dimension format for the output image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Unset: Use the channel dimension format of the input image. - input_data_format (`ChannelDimension` or `str`, *optional*): - The channel dimension format for the input image. If unset, the channel dimension format is inferred - from the input image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - - """ # noqa: E501 - min_pixels = min_pixels if min_pixels is not None else self.min_pixels - max_pixels = max_pixels if max_pixels is not None else self.max_pixels - - if size is not None: - if "shortest_edge" not in size or "longest_edge" not in size: - raise ValueError( - "size must contain 'shortest_edge' and 'longest_edge' keys." - ) - min_pixels = size["shortest_edge"] - elif min_pixels is not None and max_pixels is not None: - # backward compatibility: override size with min_pixels and max_pixels - # if they are provided. - size = {"shortest_edge": min_pixels, "longest_edge": max_pixels} - else: - size = {**self.size} - - do_resize = do_resize if do_resize is not None else self.do_resize - - resample = resample if resample is not None else self.resample - do_rescale = do_rescale if do_rescale is not None else self.do_rescale - rescale_factor = ( - rescale_factor if rescale_factor is not None else self.rescale_factor - ) - do_normalize = do_normalize if do_normalize is not None else self.do_normalize - image_mean = image_mean if image_mean is not None else self.image_mean - image_std = image_std if image_std is not None else self.image_std - patch_size = patch_size if patch_size is not None else self.patch_size - temporal_patch_size = ( - temporal_patch_size - if temporal_patch_size is not None - else self.temporal_patch_size - ) - merge_size = merge_size if merge_size is not None else self.merge_size - do_convert_rgb = ( - do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - ) - - if images is not None: - images = make_flat_list_of_images(images) - - if images is not None and not valid_images(images): - raise ValueError( - "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " - "torch.Tensor, tf.Tensor or jax.ndarray." - ) - - validate_preprocess_arguments( - rescale_factor=rescale_factor, - do_normalize=do_normalize, - image_mean=image_mean, - image_std=image_std, - do_resize=do_resize, - size=size, - resample=resample, - ) - - data = {} - if images is not None: - pixel_values, vision_grid_thws = [], [] - for image in images: - patches, image_grid_thw = self._preprocess( - image, - do_resize=do_resize, - size=size, - resample=resample, - do_rescale=do_rescale, - rescale_factor=rescale_factor, - do_normalize=do_normalize, - image_mean=image_mean, - image_std=image_std, - patch_size=patch_size, - temporal_patch_size=temporal_patch_size, - merge_size=merge_size, - data_format=data_format, - do_convert_rgb=do_convert_rgb, - input_data_format=input_data_format, - ) - pixel_values.extend(patches) - vision_grid_thws.append(image_grid_thw) - pixel_values = np.array(pixel_values) - vision_grid_thws = np.array(vision_grid_thws) - data.update( - {"pixel_values": pixel_values, "image_grid_thw": vision_grid_thws} - ) - - # kept for BC only and should be removed after v5.0 - if videos is not None: - logger.warning( - "`HunYuanVLV1ImageProcessor` works only with image inputs " - "and doesn't process videos anymore. " - "This is a deprecated behavior and will be removed in v5.0. " - "Your videos should be forwarded to `HunYuanVLV1VideoProcessor`. " - ) - videos = make_batched_videos(videos) - pixel_values_videos, vision_grid_thws_videos = [], [] - for images in videos: - patches, video_grid_thw = self._preprocess( - images, - do_resize=do_resize, - size=size, - resample=resample, - do_rescale=do_rescale, - rescale_factor=rescale_factor, - do_normalize=do_normalize, - image_mean=image_mean, - image_std=image_std, - patch_size=patch_size, - temporal_patch_size=temporal_patch_size, - merge_size=merge_size, - data_format=data_format, - do_convert_rgb=do_convert_rgb, - input_data_format=input_data_format, - ) - pixel_values_videos.extend(patches) - vision_grid_thws_videos.append(video_grid_thw) - data.update( - { - "pixel_values_videos": np.array(pixel_values_videos), - "video_grid_thw": np.array(vision_grid_thws_videos), - } - ) - - return BatchFeature(data=data, tensor_type=return_tensors) - - def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None): - """ - A utility that returns number of image patches for a given image size. - - Args: - height (`int`): - Height of the input image. - width (`int`): - Width of the input image. - images_kwargs (`dict`, *optional*): - Any kwargs to override defaults of the image processor. - Returns: - `int`: Number of image patches per image. - """ - min_pixels = ( - images_kwargs["min_pixels"] - if "min_pixels" in images_kwargs - else self.size["shortest_edge"] - ) - max_pixels = ( - images_kwargs["max_pixels"] - if "max_pixels" in images_kwargs - else self.size["longest_edge"] - ) - patch_size = images_kwargs.get("patch_size", self.patch_size) - merge_size = images_kwargs.get("merge_size", self.merge_size) - - factor = patch_size * merge_size - resized_height, resized_width = smart_resize( - height, width, factor, min_pixels=min_pixels, max_pixels=max_pixels - ) - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size - return grid_h * (grid_w + 1) + 2 - - -AutoImageProcessor.register("HunYuanVLImageProcessor", HunYuanVLImageProcessor)