minicpmv4_6: fix ImageSize (W,H) order for placeholder token calculation (#45244)
Signed-off-by: tc-mb <tianchi_cai@icloud.com>
This commit is contained in:
@@ -598,50 +598,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
||||
if version == (2, 0) or version == (2, 5):
|
||||
return image_processor.get_slice_image_placeholder(image_size)
|
||||
|
||||
if version == (4, 6):
|
||||
if max_slice_nums is None:
|
||||
max_slice_nums = image_processor.max_slice_nums
|
||||
grids = image_processor.get_sliced_grid(
|
||||
image_size,
|
||||
max_slice_nums=max_slice_nums,
|
||||
)
|
||||
patch_size = image_processor.patch_size
|
||||
scale_resolution = image_processor.scale_resolution
|
||||
|
||||
allow_upscale = grids is None
|
||||
best_size = image_processor.find_best_resize(
|
||||
image_size,
|
||||
scale_resolution,
|
||||
patch_size,
|
||||
allow_upscale=allow_upscale,
|
||||
)
|
||||
h_patches = best_size[1] // patch_size
|
||||
w_patches = best_size[0] // patch_size
|
||||
source_image_visual_tokens = (h_patches // 4) * (w_patches // 4)
|
||||
|
||||
if grids is not None:
|
||||
refine_size = image_processor.get_refine_size(
|
||||
image_size,
|
||||
grids,
|
||||
scale_resolution,
|
||||
patch_size,
|
||||
allow_upscale=True,
|
||||
)
|
||||
pw = refine_size[0] // grids[0]
|
||||
ph = refine_size[1] // grids[1]
|
||||
patch_visual_tokens = (ph // patch_size // 4) * (pw // patch_size // 4)
|
||||
else:
|
||||
patch_visual_tokens = source_image_visual_tokens
|
||||
|
||||
return image_processor.get_slice_image_placeholder(
|
||||
grids if grids is not None else [0, 0],
|
||||
image_idx=image_idx,
|
||||
max_slice_nums=max_slice_nums,
|
||||
use_image_id=use_image_id,
|
||||
source_image_visual_tokens=source_image_visual_tokens,
|
||||
patch_visual_tokens=patch_visual_tokens,
|
||||
)
|
||||
|
||||
return image_processor.get_slice_image_placeholder(
|
||||
image_size,
|
||||
image_idx=image_idx,
|
||||
@@ -675,44 +631,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
||||
max_slice_nums: int | None = None,
|
||||
) -> int:
|
||||
image_processor = self.get_image_processor()
|
||||
version = self.get_model_version()
|
||||
|
||||
grid = self.get_sliced_grid(
|
||||
image_size,
|
||||
max_slice_nums=max_slice_nums,
|
||||
)
|
||||
|
||||
if version == (4, 6):
|
||||
patch_size = image_processor.patch_size
|
||||
scale_resolution = image_processor.scale_resolution
|
||||
|
||||
allow_upscale = grid is None
|
||||
best_size = image_processor.find_best_resize(
|
||||
image_size,
|
||||
scale_resolution,
|
||||
patch_size,
|
||||
allow_upscale=allow_upscale,
|
||||
)
|
||||
h_p = best_size[1] // patch_size
|
||||
w_p = best_size[0] // patch_size
|
||||
source_tokens = (h_p // 4) * (w_p // 4)
|
||||
|
||||
if grid is None:
|
||||
return source_tokens
|
||||
|
||||
refine_size = image_processor.get_refine_size(
|
||||
image_size,
|
||||
grid,
|
||||
scale_resolution,
|
||||
patch_size,
|
||||
allow_upscale=True,
|
||||
)
|
||||
pw = refine_size[0] // grid[0]
|
||||
ph = refine_size[1] // grid[1]
|
||||
patch_tokens = (ph // patch_size // 4) * (pw // patch_size // 4)
|
||||
ncols, nrows = grid
|
||||
return source_tokens + ncols * nrows * patch_tokens
|
||||
|
||||
if grid is None:
|
||||
ncols = nrows = 0
|
||||
else:
|
||||
|
||||
@@ -509,22 +509,25 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
|
||||
downsample_mode = self._get_downsample_mode(downsample_mode)
|
||||
token_divisor = 4 if downsample_mode == "4x" else 16
|
||||
|
||||
# vLLM ImageSize is (width, height); transformers expects (height, width)
|
||||
hf_image_size = (image_size.height, image_size.width)
|
||||
|
||||
# transformers v5.7+ requires `scale_resolution` arg
|
||||
try:
|
||||
grids = image_processor.get_sliced_grid(
|
||||
image_size,
|
||||
hf_image_size,
|
||||
max_slice_nums,
|
||||
scale_res,
|
||||
)
|
||||
except TypeError:
|
||||
grids = image_processor.get_sliced_grid(
|
||||
image_size,
|
||||
hf_image_size,
|
||||
max_slice_nums,
|
||||
)
|
||||
|
||||
if grids is None:
|
||||
best_size = image_processor.find_best_resize(
|
||||
image_size,
|
||||
hf_image_size,
|
||||
scale_res,
|
||||
patch_size,
|
||||
allow_upscale=True,
|
||||
@@ -535,7 +538,7 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
|
||||
return [0, 0], source_tokens, 0
|
||||
|
||||
best_resize = image_processor.find_best_resize(
|
||||
image_size,
|
||||
hf_image_size,
|
||||
scale_res,
|
||||
patch_size,
|
||||
)
|
||||
@@ -543,7 +546,7 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
|
||||
best_resize[0] * best_resize[1] // (patch_size * patch_size * token_divisor)
|
||||
)
|
||||
refine_size = image_processor.get_refine_size(
|
||||
image_size,
|
||||
hf_image_size,
|
||||
grids,
|
||||
scale_res,
|
||||
patch_size,
|
||||
|
||||
Reference in New Issue
Block a user