[Bugfix][MiniCPM-V] Route video_embeds to the shared vision parser (#54633)

Signed-off-by: subhash polisetti <subhashr161347@gmail.com>

tests/model_executor/test_minicpmv.py

@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+import torch
+
+from vllm.model_executor.models.minicpmv import MiniCPMV2_6
+from vllm.model_executor.models.minicpmv4_6 import (
+    MiniCPMV4_6ForConditionalGeneration,
+)
+
+pytestmark = pytest.mark.skip_global_cleanup
+
+
+def make_model() -> MiniCPMV2_6:
+    return object.__new__(MiniCPMV2_6)
+
+
+def make_model_4_6() -> MiniCPMV4_6ForConditionalGeneration:
+    model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
+    model._process_vision_input = lambda vision_input, use_vit_merger=None: [
+        vision_input["image_embeds"]
+    ]
+    return model
+
+
+def test_video_embeds_reach_the_vision_parser():
+    embeds = torch.arange(32, dtype=torch.float32).reshape(1, 4, 8)
+
+    modalities = make_model()._parse_and_validate_multimodal_inputs(video_embeds=embeds)
+
+    video_input = modalities["videos"]
+    assert video_input is not None
+    assert video_input["type"] == "image_embeds"
+    assert torch.equal(video_input["image_embeds"], embeds)
+
+
+def test_video_embeds_are_embedded_by_4_6():
+    embeds = torch.arange(32, dtype=torch.float32).reshape(1, 4, 8)
+
+    embeddings = make_model_4_6().embed_multimodal(video_embeds=embeds)
+
+    assert len(embeddings) == 1
+    assert torch.equal(embeddings[0], embeds)
+
+
+def test_image_and_video_embeds_stay_in_their_own_modality():
+    image_embeds = torch.zeros(1, 4, 8)
+    video_embeds = torch.ones(1, 4, 8)
+
+    modalities = make_model()._parse_and_validate_multimodal_inputs(
+        image_embeds=image_embeds,
+        video_embeds=video_embeds,
+    )
+
+    assert torch.equal(modalities["images"]["image_embeds"], image_embeds)
+    assert torch.equal(modalities["videos"]["image_embeds"], video_embeds)

vllm/model_executor/models/minicpmv.py

@@ -469,6 +469,22 @@ def get_version_by_config(config: PretrainedConfig) -> tuple[int, ...]:
     return tuple(int(x) for x in version_str.split("."))
 
 
+_VIDEO_TO_IMAGE_KWARGS = {
+    "video_pixel_values": "pixel_values",
+    "video_image_sizes": "image_sizes",
+    "video_tgt_sizes": "tgt_sizes",
+    "video_embeds": "image_embeds",
+}
+
+
+def _image_kwargs_from_video(kwargs: Mapping[str, object]) -> dict[str, object]:
+    return {
+        _VIDEO_TO_IMAGE_KWARGS[k]: v
+        for k, v in kwargs.items()
+        if k in _VIDEO_TO_IMAGE_KWARGS
+    }
+
+
 def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
     return dict(
         pixel_values=MultiModalFieldConfig.batched("image"),
@@ -1276,7 +1292,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
                 and "videos" not in modalities
             ):
                 modalities["videos"] = self._parse_and_validate_vision_input(
-                    "videos", **{k.removeprefix("video_"): v for k, v in kwargs.items()}
+                    "videos", **_image_kwargs_from_video(kwargs)
                 )
 
         return modalities

vllm/model_executor/models/minicpmv4_6.py

@@ -59,6 +59,7 @@ from .minicpmv import (
     MiniCPMVMultiModalProcessor,
     MiniCPMVProcessingInfo,
     MiniCPMVVideoEmbeddingItems,
+    _image_kwargs_from_video,
 )
 from .module_mapping import MultiModelKeys
 from .qwen3_5 import Qwen3_5ForCausalLM
@@ -1200,18 +1201,12 @@ class MiniCPMV4_6ForConditionalGeneration(
                     for t in use_vit_merger_tensors
                 )
 
-        # Split kwargs into image / video buckets (videos are processed via
-        # the same vision pipeline; their fields just carry a ``video_`` prefix).
         image_kwargs = {
             k: v
             for k, v in kwargs.items()
             if k in ("pixel_values", "image_embeds", "tgt_sizes")
         }
-        video_kwargs = {
-            k.removeprefix("video_"): v
-            for k, v in kwargs.items()
-            if k.startswith("video_")
-        }
+        video_kwargs = _image_kwargs_from_video(kwargs)
 
         multimodal_embeddings: tuple[torch.Tensor, ...] = ()