[Bugfix][MM] Fix MiniCPM-o image processor reuse on Transformers v5 (#54501)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>

tests/models/multimodal/processing/test_minicpmv.py

@@ -7,6 +7,7 @@ import numpy as np
 import pytest
 
 from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.parse import ImageSize
 
 from ...utils import build_model_context
 
@@ -27,21 +28,36 @@ def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
 
 
 @pytest.mark.parametrize(
-    "model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")]
+    "model_ids",
+    [
+        ("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4"),
+        ("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-o-2_6"),
+    ],
 )
-def test_image_processor_for_dif_model(model_ids):
-    model_id_25, model_id_4 = model_ids
+def test_image_processor_for_different_models(model_ids):
+    first_model_id, second_model_id = model_ids
 
-    ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1})
-    processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config)
-    image_processor_25 = processor_25.info.get_image_processor()
+    first_ctx = build_model_context(
+        first_model_id,
+        limit_mm_per_prompt={"image": 1},
+    )
+    first_processor = MULTIMODAL_REGISTRY.create_processor(first_ctx.model_config)
+    first_image_processor = first_processor.info.get_image_processor()
 
-    ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1})
-    processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config)
-    image_processor_4 = processor_4.info.get_image_processor()
+    second_ctx = build_model_context(
+        second_model_id,
+        limit_mm_per_prompt={"image": 1},
+    )
+    second_processor = MULTIMODAL_REGISTRY.create_processor(second_ctx.model_config)
+    second_image_processor = second_processor.info.get_image_processor()
 
-    assert type(image_processor_25) is not type(image_processor_4)
-    assert type(image_processor_25).__module__ != type(image_processor_4).__module__
+    second_processor.info.get_sliced_grid(ImageSize(width=128, height=128))
+
+    assert type(first_image_processor) is not type(second_image_processor)
+    assert (
+        type(first_image_processor).__module__
+        != type(second_image_processor).__module__
+    )
 
 
 @pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])

vllm/model_executor/models/minicpmo.py

@@ -282,28 +282,28 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
     def get_hf_processor(self, **kwargs: object) -> "MiniCPMOProcessor":
         """Get vendored MiniCPMOProcessor for multimodal (image+audio) inputs.
 
-        Creates a vendored processor that reuses the HF image processor,
-        feature extractor, and tokenizer; applies the correct audio pooling
-        configuration; and converts numpy arrays in the image processor to
-        lists for serialization compatibility. The returned processor is
+        Creates a vendored processor that uses the checkpoint-specific HF image
+        processor, feature extractor, and tokenizer; applies the correct audio
+        pooling configuration; and converts numpy arrays in the image processor
+        to lists for serialization compatibility. The returned processor is
         compatible with Transformers v5.
         """
         import numpy as np
 
         hf_processor = self.ctx.get_hf_processor(**kwargs)
+        image_processor = self._get_checkpoint_image_processor(**kwargs)
 
         from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor
 
         # Create vendored processor with correct configuration
         vendored_processor = MiniCPMOProcessor(
-            image_processor=hf_processor.image_processor,
+            image_processor=image_processor,
             feature_extractor=hf_processor.feature_extractor,
             tokenizer=hf_processor.tokenizer,
             pool_step=self.get_default_audio_pool_step(),
         )
 
         # Convert numpy arrays in image processor to lists for serialization
-        image_processor = vendored_processor.image_processor
         for attr in ("mean", "std"):
             val = getattr(image_processor, attr, None)
             if val is not None and isinstance(val, np.ndarray):

vllm/model_executor/models/minicpmv.py

@@ -611,16 +611,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
     def get_hf_config(self):
         return self.ctx.get_hf_config()
 
-    def get_hf_processor(self, **kwargs: object):
+    def _get_checkpoint_image_processor(self, **kwargs: object):
         model_config = self.ctx.model_config
         processor_cls = self._image_processor_cls
         merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs)
 
-        # AutoProcessor only for tokenizer; its image_processor is resolved by
-        # class name and can pick the wrong checkpoint across MiniCPM-V versions.
-        hf_processor = self.ctx.get_hf_processor(**kwargs)
-
-        image_processor = cached_get_image_processor(
+        return cached_get_image_processor(
             model_config.model,
             revision=model_config.revision,
             trust_remote_code=model_config.trust_remote_code,
@@ -628,6 +624,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
             **merged_kwargs,
         )
 
+    def get_hf_processor(self, **kwargs: object):
+        # AutoProcessor only for tokenizer; its image_processor is resolved by
+        # class name and can pick the wrong MiniCPM checkpoint.
+        hf_processor = self.ctx.get_hf_processor(**kwargs)
+        image_processor = self._get_checkpoint_image_processor(**kwargs)
+
         from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
 
         vendored_processor = MiniCPMVProcessor(