[Model] Support MiniCPM-V 4.6 (#41254)
Signed-off-by: caitianchi <caitianchi@tc-mb.com> Signed-off-by: tc-mb <157115220+tc-mb@users.noreply.github.com> Co-authored-by: caitianchi <caitianchi@tc-mb.com>
This commit is contained in:
@@ -1098,6 +1098,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
|||||||
},
|
},
|
||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
),
|
),
|
||||||
|
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
|
||||||
|
"openbmb/MiniCPM-V-4_6",
|
||||||
|
min_transformers_version="5.7.0",
|
||||||
|
),
|
||||||
"MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo(
|
"MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo(
|
||||||
"MiniMaxAI/MiniMax-VL-01",
|
"MiniMaxAI/MiniMax-VL-01",
|
||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
|
|||||||
@@ -546,8 +546,9 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
|||||||
# NumPy arrays are considered as Iterable but not Sequence in
|
# NumPy arrays are considered as Iterable but not Sequence in
|
||||||
# https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
|
# https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
|
||||||
image_processor = hf_processor.image_processor # type: ignore
|
image_processor = hf_processor.image_processor # type: ignore
|
||||||
for attr in ("mean", "std"):
|
# transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std`
|
||||||
val = getattr(image_processor, attr)
|
for attr in ("mean", "std", "image_mean", "image_std"):
|
||||||
|
val = getattr(image_processor, attr, None)
|
||||||
if isinstance(val, np.ndarray):
|
if isinstance(val, np.ndarray):
|
||||||
setattr(image_processor, attr, val.tolist())
|
setattr(image_processor, attr, val.tolist())
|
||||||
|
|
||||||
@@ -585,6 +586,50 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
|||||||
if version == (2, 0) or version == (2, 5):
|
if version == (2, 0) or version == (2, 5):
|
||||||
return image_processor.get_slice_image_placeholder(image_size)
|
return image_processor.get_slice_image_placeholder(image_size)
|
||||||
|
|
||||||
|
if version == (4, 6):
|
||||||
|
if max_slice_nums is None:
|
||||||
|
max_slice_nums = image_processor.max_slice_nums
|
||||||
|
grids = image_processor.get_sliced_grid(
|
||||||
|
image_size,
|
||||||
|
max_slice_nums=max_slice_nums,
|
||||||
|
)
|
||||||
|
patch_size = image_processor.patch_size
|
||||||
|
scale_resolution = image_processor.scale_resolution
|
||||||
|
|
||||||
|
allow_upscale = grids is None
|
||||||
|
best_size = image_processor.find_best_resize(
|
||||||
|
image_size,
|
||||||
|
scale_resolution,
|
||||||
|
patch_size,
|
||||||
|
allow_upscale=allow_upscale,
|
||||||
|
)
|
||||||
|
h_patches = best_size[1] // patch_size
|
||||||
|
w_patches = best_size[0] // patch_size
|
||||||
|
source_image_visual_tokens = (h_patches // 4) * (w_patches // 4)
|
||||||
|
|
||||||
|
if grids is not None:
|
||||||
|
refine_size = image_processor.get_refine_size(
|
||||||
|
image_size,
|
||||||
|
grids,
|
||||||
|
scale_resolution,
|
||||||
|
patch_size,
|
||||||
|
allow_upscale=True,
|
||||||
|
)
|
||||||
|
pw = refine_size[0] // grids[0]
|
||||||
|
ph = refine_size[1] // grids[1]
|
||||||
|
patch_visual_tokens = (ph // patch_size // 4) * (pw // patch_size // 4)
|
||||||
|
else:
|
||||||
|
patch_visual_tokens = source_image_visual_tokens
|
||||||
|
|
||||||
|
return image_processor.get_slice_image_placeholder(
|
||||||
|
grids if grids is not None else [0, 0],
|
||||||
|
image_idx=image_idx,
|
||||||
|
max_slice_nums=max_slice_nums,
|
||||||
|
use_image_id=use_image_id,
|
||||||
|
source_image_visual_tokens=source_image_visual_tokens,
|
||||||
|
patch_visual_tokens=patch_visual_tokens,
|
||||||
|
)
|
||||||
|
|
||||||
return image_processor.get_slice_image_placeholder(
|
return image_processor.get_slice_image_placeholder(
|
||||||
image_size,
|
image_size,
|
||||||
image_idx=image_idx,
|
image_idx=image_idx,
|
||||||
@@ -618,11 +663,44 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
|||||||
max_slice_nums: int | None = None,
|
max_slice_nums: int | None = None,
|
||||||
) -> int:
|
) -> int:
|
||||||
image_processor = self.get_image_processor()
|
image_processor = self.get_image_processor()
|
||||||
|
version = self.get_model_version()
|
||||||
|
|
||||||
grid = self.get_sliced_grid(
|
grid = self.get_sliced_grid(
|
||||||
image_size,
|
image_size,
|
||||||
max_slice_nums=max_slice_nums,
|
max_slice_nums=max_slice_nums,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if version == (4, 6):
|
||||||
|
patch_size = image_processor.patch_size
|
||||||
|
scale_resolution = image_processor.scale_resolution
|
||||||
|
|
||||||
|
allow_upscale = grid is None
|
||||||
|
best_size = image_processor.find_best_resize(
|
||||||
|
image_size,
|
||||||
|
scale_resolution,
|
||||||
|
patch_size,
|
||||||
|
allow_upscale=allow_upscale,
|
||||||
|
)
|
||||||
|
h_p = best_size[1] // patch_size
|
||||||
|
w_p = best_size[0] // patch_size
|
||||||
|
source_tokens = (h_p // 4) * (w_p // 4)
|
||||||
|
|
||||||
|
if grid is None:
|
||||||
|
return source_tokens
|
||||||
|
|
||||||
|
refine_size = image_processor.get_refine_size(
|
||||||
|
image_size,
|
||||||
|
grid,
|
||||||
|
scale_resolution,
|
||||||
|
patch_size,
|
||||||
|
allow_upscale=True,
|
||||||
|
)
|
||||||
|
pw = refine_size[0] // grid[0]
|
||||||
|
ph = refine_size[1] // grid[1]
|
||||||
|
patch_tokens = (ph // patch_size // 4) * (pw // patch_size // 4)
|
||||||
|
ncols, nrows = grid
|
||||||
|
return source_tokens + ncols * nrows * patch_tokens
|
||||||
|
|
||||||
if grid is None:
|
if grid is None:
|
||||||
ncols = nrows = 0
|
ncols = nrows = 0
|
||||||
else:
|
else:
|
||||||
@@ -839,7 +917,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
|||||||
out_keys: set[str],
|
out_keys: set[str],
|
||||||
) -> dict[str, NestedTensors]:
|
) -> dict[str, NestedTensors]:
|
||||||
# This processor supports zipping prompt and mm_data together
|
# This processor supports zipping prompt and mm_data together
|
||||||
if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5)}:
|
if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}:
|
||||||
inputs = super()._call_hf_processor(
|
inputs = super()._call_hf_processor(
|
||||||
prompt=prompts, # type: ignore
|
prompt=prompts, # type: ignore
|
||||||
mm_data=mm_data,
|
mm_data=mm_data,
|
||||||
@@ -971,10 +1049,15 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
|||||||
if version == (2, 0) or version == (2, 5):
|
if version == (2, 0) or version == (2, 5):
|
||||||
im_start = image_processor.im_start_token
|
im_start = image_processor.im_start_token
|
||||||
im_end = image_processor.im_end_token
|
im_end = image_processor.im_end_token
|
||||||
else:
|
elif hasattr(image_processor, "im_id_start"):
|
||||||
im_start = image_processor.im_id_start
|
im_start = image_processor.im_id_start
|
||||||
im_end = image_processor.im_id_end
|
im_end = image_processor.im_id_end
|
||||||
|
else:
|
||||||
|
# transformers v5.7+ keeps im_id tokens on the tokenizer.
|
||||||
|
im_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||||
|
im_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||||
|
|
||||||
|
embed_text = getattr(tokenizer, "image_token", "<unk>")
|
||||||
new_update = new_update.with_content(
|
new_update = new_update.with_content(
|
||||||
PromptUpdateDetails.select_text(
|
PromptUpdateDetails.select_text(
|
||||||
text.replace(
|
text.replace(
|
||||||
@@ -982,7 +1065,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
|||||||
f"{im_start}{new_item_idx}{im_end}",
|
f"{im_start}{new_item_idx}{im_end}",
|
||||||
1,
|
1,
|
||||||
),
|
),
|
||||||
"<unk>",
|
embed_text,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -478,6 +478,10 @@ _MULTIMODAL_MODELS = {
|
|||||||
),
|
),
|
||||||
"MiniCPMO": ("minicpmo", "MiniCPMO"),
|
"MiniCPMO": ("minicpmo", "MiniCPMO"),
|
||||||
"MiniCPMV": ("minicpmv", "MiniCPMV"),
|
"MiniCPMV": ("minicpmv", "MiniCPMV"),
|
||||||
|
"MiniCPMV4_6ForConditionalGeneration": (
|
||||||
|
"minicpmv4_6",
|
||||||
|
"MiniCPMV4_6ForConditionalGeneration",
|
||||||
|
),
|
||||||
"Mistral3ForConditionalGeneration": (
|
"Mistral3ForConditionalGeneration": (
|
||||||
"mistral3",
|
"mistral3",
|
||||||
"Mistral3ForConditionalGeneration",
|
"Mistral3ForConditionalGeneration",
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
|
|||||||
"deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
|
"deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
|
||||||
"fuyu": CHAT_TEMPLATES_DIR / "template_fuyu.jinja",
|
"fuyu": CHAT_TEMPLATES_DIR / "template_fuyu.jinja",
|
||||||
"minicpmv": _get_minicpmv_chat_template_fallback,
|
"minicpmv": _get_minicpmv_chat_template_fallback,
|
||||||
|
"minicpmv4_6": _get_minicpmv_chat_template_fallback,
|
||||||
"paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja",
|
"paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja",
|
||||||
"qwen": _get_qwen_chat_template_fallback,
|
"qwen": _get_qwen_chat_template_fallback,
|
||||||
"siglip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
|
"siglip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
|
||||||
|
|||||||
Reference in New Issue
Block a user