[Bugfix] Standardize getting number of image patches/tokens (#34358)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -10,7 +10,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import BaseImageProcessor, PretrainedConfig
|
||||
from transformers.activations import GELUActivation
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
|
||||
@@ -1011,24 +1011,25 @@ class KeyeProcessingInfo(BaseProcessingInfo):
|
||||
image_height: int,
|
||||
num_frames: int = 1,
|
||||
do_resize: bool = True,
|
||||
image_processor,
|
||||
image_processor: BaseImageProcessor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[ImageSize, int]:
|
||||
if image_processor is None:
|
||||
image_processor = self.get_image_processor()
|
||||
|
||||
hf_config = self.get_hf_config()
|
||||
vision_config = hf_config.vision_config
|
||||
patch_size = vision_config.patch_size
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
temporal_patch_size = 1
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
|
||||
size = mm_kwargs.get("size", image_processor.size)
|
||||
|
||||
if do_resize:
|
||||
resized_height, resized_width = smart_resize(
|
||||
height=image_height,
|
||||
width=image_width,
|
||||
factor=patch_size * merge_size,
|
||||
min_pixels=image_processor.min_pixels,
|
||||
max_pixels=image_processor.max_pixels,
|
||||
min_pixels=size["min_pixels"],
|
||||
max_pixels=size["max_pixels"],
|
||||
)
|
||||
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
|
||||
else:
|
||||
@@ -1050,12 +1051,14 @@ class KeyeProcessingInfo(BaseProcessingInfo):
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
image_processor,
|
||||
image_processor: BaseImageProcessor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> int:
|
||||
_, num_image_tokens = self._get_vision_info(
|
||||
image_width=image_width,
|
||||
image_height=image_height,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
return num_image_tokens
|
||||
|
||||
@@ -1065,36 +1068,42 @@ class KeyeProcessingInfo(BaseProcessingInfo):
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
num_frames: int,
|
||||
image_processor,
|
||||
image_processor: BaseImageProcessor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> int:
|
||||
_, num_video_tokens = self._get_vision_info(
|
||||
image_width=image_width,
|
||||
image_height=image_height,
|
||||
num_frames=num_frames,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs=mm_kwargs,
|
||||
)
|
||||
return num_video_tokens
|
||||
|
||||
def get_image_size_with_most_features(
|
||||
self,
|
||||
) -> ImageSize:
|
||||
def get_image_size_with_most_features(self) -> ImageSize:
|
||||
image_processor = self.get_image_processor()
|
||||
|
||||
max_image_size, _ = self._get_vision_info(
|
||||
image_width=self.get_max_image_size(),
|
||||
image_height=self.get_max_image_size(),
|
||||
image_processor=None,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs={},
|
||||
)
|
||||
return max_image_size
|
||||
|
||||
def get_max_image_tokens(self) -> int:
|
||||
image_processor = self.get_image_processor()
|
||||
target_width, target_height = self.get_image_size_with_most_features()
|
||||
|
||||
return self.get_num_image_tokens(
|
||||
image_width=target_width,
|
||||
image_height=target_height,
|
||||
image_processor=None,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs={},
|
||||
)
|
||||
|
||||
def _get_max_video_frames(self, max_tokens: int) -> int:
|
||||
image_processor = self.get_image_processor()
|
||||
target_width, target_height = self.get_image_size_with_most_features()
|
||||
|
||||
num_frames = 0
|
||||
@@ -1105,7 +1114,8 @@ class KeyeProcessingInfo(BaseProcessingInfo):
|
||||
image_width=target_width,
|
||||
image_height=target_height,
|
||||
num_frames=next_num_frames,
|
||||
image_processor=None,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs={},
|
||||
)
|
||||
|
||||
if next_max_tokens > max_tokens:
|
||||
@@ -1130,13 +1140,15 @@ class KeyeProcessingInfo(BaseProcessingInfo):
|
||||
return max(max_frames_per_video, 1)
|
||||
|
||||
def get_max_video_tokens(self, seq_len: int) -> int:
|
||||
image_processor = self.get_image_processor()
|
||||
target_width, target_height = self.get_image_size_with_most_features()
|
||||
|
||||
return self.get_num_video_tokens(
|
||||
image_width=target_width,
|
||||
image_height=target_height,
|
||||
num_frames=self.get_num_frames_with_most_features(seq_len),
|
||||
image_processor=None,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user