[Bugfix] Standardize getting number of image patches/tokens (#34358)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-02-12 20:47:01 -08:00
committed by GitHub
parent 6afa587d31
commit 372b2e762a
29 changed files with 319 additions and 331 deletions
+12 -8
View File
@@ -23,7 +23,7 @@ import numpy as np
import torch
import torch.nn as nn
from einops import rearrange
from transformers import BatchFeature, PretrainedConfig
from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig
from transformers.activations import GELUActivation
from transformers.modeling_outputs import (
BaseModelOutputWithPooling,
@@ -147,21 +147,23 @@ class PaddleOCRVLProcessingInfo(BaseProcessingInfo):
*,
image_width: int,
image_height: int,
image_processor,
image_processor: BaseImageProcessor,
mm_kwargs: Mapping[str, object],
) -> int:
if image_processor is None:
image_processor = self.get_image_processor()
hf_config = self.get_hf_config()
vision_config = hf_config.vision_config
patch_size = vision_config.patch_size
merge_size = vision_config.spatial_merge_size
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
size = mm_kwargs.get("size", image_processor.size)
resized_height, resized_width = smart_resize(
height=image_height,
width=image_width,
factor=patch_size * merge_size,
min_pixels=image_processor.min_pixels,
max_pixels=image_processor.max_pixels,
min_pixels=size["min_pixels"],
max_pixels=size["max_pixels"],
)
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
@@ -176,12 +178,13 @@ class PaddleOCRVLProcessingInfo(BaseProcessingInfo):
def get_image_size_with_most_features(self) -> ImageSize:
hf_config = self.get_hf_config()
image_processor = self.get_image_processor()
# See `smart_resize` for the calculation of the image size.
merge_size = hf_config.vision_config.spatial_merge_size
patch_size = hf_config.vision_config.patch_size
factor = merge_size * patch_size
max_num_tokens = self.get_image_processor().max_pixels // (factor**2)
max_num_tokens = image_processor.max_pixels // (factor**2)
# Find factors of max_num_tokens close to its square root
# to create a dummy image with a reasonable aspect ratio.
h_patches = int(math.sqrt(max_num_tokens))
@@ -276,6 +279,7 @@ class PaddleOCRVLMultiModalProcessor(
image_width=image_size.width,
image_height=image_size.height,
image_processor=image_processor,
mm_kwargs=hf_processor_mm_kwargs,
)
return [image_token_id] * num_image_tokens