[Model] Support MiniCPM-V 4.6 (#41254)

Signed-off-by: caitianchi <caitianchi@tc-mb.com>
Signed-off-by: tc-mb <157115220+tc-mb@users.noreply.github.com>
Co-authored-by: caitianchi <caitianchi@tc-mb.com>
This commit is contained in:
tc-mb
2026-05-12 14:28:10 +00:00
committed by GitHub
co-authored by caitianchi
parent d077622d60
commit 7a9cc5e7f0
5 changed files with 1302 additions and 5 deletions
+4
View File
@@ -1098,6 +1098,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
},
trust_remote_code=True,
),
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
"openbmb/MiniCPM-V-4_6",
min_transformers_version="5.7.0",
),
"MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo(
"MiniMaxAI/MiniMax-VL-01",
trust_remote_code=True,
+88 -5
View File
@@ -546,8 +546,9 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
# NumPy arrays are considered as Iterable but not Sequence in
# https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
image_processor = hf_processor.image_processor # type: ignore
for attr in ("mean", "std"):
val = getattr(image_processor, attr)
# transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std`
for attr in ("mean", "std", "image_mean", "image_std"):
val = getattr(image_processor, attr, None)
if isinstance(val, np.ndarray):
setattr(image_processor, attr, val.tolist())
@@ -585,6 +586,50 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
if version == (2, 0) or version == (2, 5):
return image_processor.get_slice_image_placeholder(image_size)
if version == (4, 6):
if max_slice_nums is None:
max_slice_nums = image_processor.max_slice_nums
grids = image_processor.get_sliced_grid(
image_size,
max_slice_nums=max_slice_nums,
)
patch_size = image_processor.patch_size
scale_resolution = image_processor.scale_resolution
allow_upscale = grids is None
best_size = image_processor.find_best_resize(
image_size,
scale_resolution,
patch_size,
allow_upscale=allow_upscale,
)
h_patches = best_size[1] // patch_size
w_patches = best_size[0] // patch_size
source_image_visual_tokens = (h_patches // 4) * (w_patches // 4)
if grids is not None:
refine_size = image_processor.get_refine_size(
image_size,
grids,
scale_resolution,
patch_size,
allow_upscale=True,
)
pw = refine_size[0] // grids[0]
ph = refine_size[1] // grids[1]
patch_visual_tokens = (ph // patch_size // 4) * (pw // patch_size // 4)
else:
patch_visual_tokens = source_image_visual_tokens
return image_processor.get_slice_image_placeholder(
grids if grids is not None else [0, 0],
image_idx=image_idx,
max_slice_nums=max_slice_nums,
use_image_id=use_image_id,
source_image_visual_tokens=source_image_visual_tokens,
patch_visual_tokens=patch_visual_tokens,
)
return image_processor.get_slice_image_placeholder(
image_size,
image_idx=image_idx,
@@ -618,11 +663,44 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
max_slice_nums: int | None = None,
) -> int:
image_processor = self.get_image_processor()
version = self.get_model_version()
grid = self.get_sliced_grid(
image_size,
max_slice_nums=max_slice_nums,
)
if version == (4, 6):
patch_size = image_processor.patch_size
scale_resolution = image_processor.scale_resolution
allow_upscale = grid is None
best_size = image_processor.find_best_resize(
image_size,
scale_resolution,
patch_size,
allow_upscale=allow_upscale,
)
h_p = best_size[1] // patch_size
w_p = best_size[0] // patch_size
source_tokens = (h_p // 4) * (w_p // 4)
if grid is None:
return source_tokens
refine_size = image_processor.get_refine_size(
image_size,
grid,
scale_resolution,
patch_size,
allow_upscale=True,
)
pw = refine_size[0] // grid[0]
ph = refine_size[1] // grid[1]
patch_tokens = (ph // patch_size // 4) * (pw // patch_size // 4)
ncols, nrows = grid
return source_tokens + ncols * nrows * patch_tokens
if grid is None:
ncols = nrows = 0
else:
@@ -839,7 +917,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
out_keys: set[str],
) -> dict[str, NestedTensors]:
# This processor supports zipping prompt and mm_data together
if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5)}:
if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}:
inputs = super()._call_hf_processor(
prompt=prompts, # type: ignore
mm_data=mm_data,
@@ -971,10 +1049,15 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
if version == (2, 0) or version == (2, 5):
im_start = image_processor.im_start_token
im_end = image_processor.im_end_token
else:
elif hasattr(image_processor, "im_id_start"):
im_start = image_processor.im_id_start
im_end = image_processor.im_id_end
else:
# transformers v5.7+ keeps im_id tokens on the tokenizer.
im_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
im_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
embed_text = getattr(tokenizer, "image_token", "<unk>")
new_update = new_update.with_content(
PromptUpdateDetails.select_text(
text.replace(
@@ -982,7 +1065,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
f"{im_start}{new_item_idx}{im_end}",
1,
),
"<unk>",
embed_text,
)
)
File diff suppressed because it is too large Load Diff
+4
View File
@@ -478,6 +478,10 @@ _MULTIMODAL_MODELS = {
),
"MiniCPMO": ("minicpmo", "MiniCPMO"),
"MiniCPMV": ("minicpmv", "MiniCPMV"),
"MiniCPMV4_6ForConditionalGeneration": (
"minicpmv4_6",
"MiniCPMV4_6ForConditionalGeneration",
),
"Mistral3ForConditionalGeneration": (
"mistral3",
"Mistral3ForConditionalGeneration",
@@ -39,6 +39,7 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
"deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
"fuyu": CHAT_TEMPLATES_DIR / "template_fuyu.jinja",
"minicpmv": _get_minicpmv_chat_template_fallback,
"minicpmv4_6": _get_minicpmv_chat_template_fallback,
"paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja",
"qwen": _get_qwen_chat_template_fallback,
"siglip": CHAT_TEMPLATES_DIR / "template_basic.jinja",