[Bugfix][MiniCPM-V] Route video_embeds to the shared vision parser (#54633)

Signed-off-by: subhash polisetti <subhashr161347@gmail.com>
This commit is contained in:
Subhash Polisetti
2026-09-01 09:16:07 +00:00
committed by GitHub
parent 5414b4e694
commit 4707679cd2
3 changed files with 75 additions and 8 deletions
+17 -1
View File
@@ -469,6 +469,22 @@ def get_version_by_config(config: PretrainedConfig) -> tuple[int, ...]:
return tuple(int(x) for x in version_str.split("."))
_VIDEO_TO_IMAGE_KWARGS = {
"video_pixel_values": "pixel_values",
"video_image_sizes": "image_sizes",
"video_tgt_sizes": "tgt_sizes",
"video_embeds": "image_embeds",
}
def _image_kwargs_from_video(kwargs: Mapping[str, object]) -> dict[str, object]:
return {
_VIDEO_TO_IMAGE_KWARGS[k]: v
for k, v in kwargs.items()
if k in _VIDEO_TO_IMAGE_KWARGS
}
def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
return dict(
pixel_values=MultiModalFieldConfig.batched("image"),
@@ -1276,7 +1292,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
and "videos" not in modalities
):
modalities["videos"] = self._parse_and_validate_vision_input(
"videos", **{k.removeprefix("video_"): v for k, v in kwargs.items()}
"videos", **_image_kwargs_from_video(kwargs)
)
return modalities
+2 -7
View File
@@ -59,6 +59,7 @@ from .minicpmv import (
MiniCPMVMultiModalProcessor,
MiniCPMVProcessingInfo,
MiniCPMVVideoEmbeddingItems,
_image_kwargs_from_video,
)
from .module_mapping import MultiModelKeys
from .qwen3_5 import Qwen3_5ForCausalLM
@@ -1200,18 +1201,12 @@ class MiniCPMV4_6ForConditionalGeneration(
for t in use_vit_merger_tensors
)
# Split kwargs into image / video buckets (videos are processed via
# the same vision pipeline; their fields just carry a ``video_`` prefix).
image_kwargs = {
k: v
for k, v in kwargs.items()
if k in ("pixel_values", "image_embeds", "tgt_sizes")
}
video_kwargs = {
k.removeprefix("video_"): v
for k, v in kwargs.items()
if k.startswith("video_")
}
video_kwargs = _image_kwargs_from_video(kwargs)
multimodal_embeddings: tuple[torch.Tensor, ...] = ()