[Bugfix][MM] Fix MiniCPM-o image processor reuse on Transformers v5 (#54501)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
This commit is contained in:
Andreas Karatzas
2026-08-31 17:07:52 +08:00
committed by GitHub
parent 5bfd76372d
commit 399247cc88
3 changed files with 41 additions and 23 deletions
@@ -7,6 +7,7 @@ import numpy as np
import pytest import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import ImageSize
from ...utils import build_model_context from ...utils import build_model_context
@@ -27,21 +28,36 @@ def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
@pytest.mark.parametrize( @pytest.mark.parametrize(
"model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")] "model_ids",
[
("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4"),
("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-o-2_6"),
],
) )
def test_image_processor_for_dif_model(model_ids): def test_image_processor_for_different_models(model_ids):
model_id_25, model_id_4 = model_ids first_model_id, second_model_id = model_ids
ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1}) first_ctx = build_model_context(
processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config) first_model_id,
image_processor_25 = processor_25.info.get_image_processor() limit_mm_per_prompt={"image": 1},
)
first_processor = MULTIMODAL_REGISTRY.create_processor(first_ctx.model_config)
first_image_processor = first_processor.info.get_image_processor()
ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1}) second_ctx = build_model_context(
processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config) second_model_id,
image_processor_4 = processor_4.info.get_image_processor() limit_mm_per_prompt={"image": 1},
)
second_processor = MULTIMODAL_REGISTRY.create_processor(second_ctx.model_config)
second_image_processor = second_processor.info.get_image_processor()
assert type(image_processor_25) is not type(image_processor_4) second_processor.info.get_sliced_grid(ImageSize(width=128, height=128))
assert type(image_processor_25).__module__ != type(image_processor_4).__module__
assert type(first_image_processor) is not type(second_image_processor)
assert (
type(first_image_processor).__module__
!= type(second_image_processor).__module__
)
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"]) @pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
+6 -6
View File
@@ -282,28 +282,28 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
def get_hf_processor(self, **kwargs: object) -> "MiniCPMOProcessor": def get_hf_processor(self, **kwargs: object) -> "MiniCPMOProcessor":
"""Get vendored MiniCPMOProcessor for multimodal (image+audio) inputs. """Get vendored MiniCPMOProcessor for multimodal (image+audio) inputs.
Creates a vendored processor that reuses the HF image processor, Creates a vendored processor that uses the checkpoint-specific HF image
feature extractor, and tokenizer; applies the correct audio pooling processor, feature extractor, and tokenizer; applies the correct audio
configuration; and converts numpy arrays in the image processor to pooling configuration; and converts numpy arrays in the image processor
lists for serialization compatibility. The returned processor is to lists for serialization compatibility. The returned processor is
compatible with Transformers v5. compatible with Transformers v5.
""" """
import numpy as np import numpy as np
hf_processor = self.ctx.get_hf_processor(**kwargs) hf_processor = self.ctx.get_hf_processor(**kwargs)
image_processor = self._get_checkpoint_image_processor(**kwargs)
from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor
# Create vendored processor with correct configuration # Create vendored processor with correct configuration
vendored_processor = MiniCPMOProcessor( vendored_processor = MiniCPMOProcessor(
image_processor=hf_processor.image_processor, image_processor=image_processor,
feature_extractor=hf_processor.feature_extractor, feature_extractor=hf_processor.feature_extractor,
tokenizer=hf_processor.tokenizer, tokenizer=hf_processor.tokenizer,
pool_step=self.get_default_audio_pool_step(), pool_step=self.get_default_audio_pool_step(),
) )
# Convert numpy arrays in image processor to lists for serialization # Convert numpy arrays in image processor to lists for serialization
image_processor = vendored_processor.image_processor
for attr in ("mean", "std"): for attr in ("mean", "std"):
val = getattr(image_processor, attr, None) val = getattr(image_processor, attr, None)
if val is not None and isinstance(val, np.ndarray): if val is not None and isinstance(val, np.ndarray):
+8 -6
View File
@@ -611,16 +611,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
def get_hf_config(self): def get_hf_config(self):
return self.ctx.get_hf_config() return self.ctx.get_hf_config()
def get_hf_processor(self, **kwargs: object): def _get_checkpoint_image_processor(self, **kwargs: object):
model_config = self.ctx.model_config model_config = self.ctx.model_config
processor_cls = self._image_processor_cls processor_cls = self._image_processor_cls
merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs) merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs)
# AutoProcessor only for tokenizer; its image_processor is resolved by return cached_get_image_processor(
# class name and can pick the wrong checkpoint across MiniCPM-V versions.
hf_processor = self.ctx.get_hf_processor(**kwargs)
image_processor = cached_get_image_processor(
model_config.model, model_config.model,
revision=model_config.revision, revision=model_config.revision,
trust_remote_code=model_config.trust_remote_code, trust_remote_code=model_config.trust_remote_code,
@@ -628,6 +624,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
**merged_kwargs, **merged_kwargs,
) )
def get_hf_processor(self, **kwargs: object):
# AutoProcessor only for tokenizer; its image_processor is resolved by
# class name and can pick the wrong MiniCPM checkpoint.
hf_processor = self.ctx.get_hf_processor(**kwargs)
image_processor = self._get_checkpoint_image_processor(**kwargs)
from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
vendored_processor = MiniCPMVProcessor( vendored_processor = MiniCPMVProcessor(