[Bugfix] Standardize getting number of image patches/tokens (#34358)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -23,7 +23,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
from transformers import BatchFeature, PretrainedConfig
|
||||
from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig
|
||||
from transformers.activations import GELUActivation
|
||||
from transformers.modeling_outputs import (
|
||||
BaseModelOutputWithPooling,
|
||||
@@ -147,21 +147,23 @@ class PaddleOCRVLProcessingInfo(BaseProcessingInfo):
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
image_processor,
|
||||
image_processor: BaseImageProcessor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> int:
|
||||
if image_processor is None:
|
||||
image_processor = self.get_image_processor()
|
||||
|
||||
hf_config = self.get_hf_config()
|
||||
vision_config = hf_config.vision_config
|
||||
patch_size = vision_config.patch_size
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
|
||||
size = mm_kwargs.get("size", image_processor.size)
|
||||
|
||||
resized_height, resized_width = smart_resize(
|
||||
height=image_height,
|
||||
width=image_width,
|
||||
factor=patch_size * merge_size,
|
||||
min_pixels=image_processor.min_pixels,
|
||||
max_pixels=image_processor.max_pixels,
|
||||
min_pixels=size["min_pixels"],
|
||||
max_pixels=size["max_pixels"],
|
||||
)
|
||||
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
|
||||
|
||||
@@ -176,12 +178,13 @@ class PaddleOCRVLProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
def get_image_size_with_most_features(self) -> ImageSize:
|
||||
hf_config = self.get_hf_config()
|
||||
image_processor = self.get_image_processor()
|
||||
|
||||
# See `smart_resize` for the calculation of the image size.
|
||||
merge_size = hf_config.vision_config.spatial_merge_size
|
||||
patch_size = hf_config.vision_config.patch_size
|
||||
factor = merge_size * patch_size
|
||||
max_num_tokens = self.get_image_processor().max_pixels // (factor**2)
|
||||
max_num_tokens = image_processor.max_pixels // (factor**2)
|
||||
# Find factors of max_num_tokens close to its square root
|
||||
# to create a dummy image with a reasonable aspect ratio.
|
||||
h_patches = int(math.sqrt(max_num_tokens))
|
||||
@@ -276,6 +279,7 @@ class PaddleOCRVLMultiModalProcessor(
|
||||
image_width=image_size.width,
|
||||
image_height=image_size.height,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
return [image_token_id] * num_image_tokens
|
||||
|
||||
Reference in New Issue
Block a user