diff --git a/tests/models/multimodal/processing/test_minicpmv.py b/tests/models/multimodal/processing/test_minicpmv.py index 9424ea3959..59cb4331f3 100644 --- a/tests/models/multimodal/processing/test_minicpmv.py +++ b/tests/models/multimodal/processing/test_minicpmv.py @@ -7,6 +7,7 @@ import numpy as np import pytest from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.parse import ImageSize from ...utils import build_model_context @@ -27,21 +28,36 @@ def test_get_hf_processor_for_same_model_different_kwargs(model_id: str): @pytest.mark.parametrize( - "model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")] + "model_ids", + [ + ("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4"), + ("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-o-2_6"), + ], ) -def test_image_processor_for_dif_model(model_ids): - model_id_25, model_id_4 = model_ids +def test_image_processor_for_different_models(model_ids): + first_model_id, second_model_id = model_ids - ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1}) - processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config) - image_processor_25 = processor_25.info.get_image_processor() + first_ctx = build_model_context( + first_model_id, + limit_mm_per_prompt={"image": 1}, + ) + first_processor = MULTIMODAL_REGISTRY.create_processor(first_ctx.model_config) + first_image_processor = first_processor.info.get_image_processor() - ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1}) - processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config) - image_processor_4 = processor_4.info.get_image_processor() + second_ctx = build_model_context( + second_model_id, + limit_mm_per_prompt={"image": 1}, + ) + second_processor = MULTIMODAL_REGISTRY.create_processor(second_ctx.model_config) + second_image_processor = second_processor.info.get_image_processor() - assert type(image_processor_25) is not type(image_processor_4) - assert type(image_processor_25).__module__ != type(image_processor_4).__module__ + second_processor.info.get_sliced_grid(ImageSize(width=128, height=128)) + + assert type(first_image_processor) is not type(second_image_processor) + assert ( + type(first_image_processor).__module__ + != type(second_image_processor).__module__ + ) @pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"]) diff --git a/vllm/model_executor/models/minicpmo.py b/vllm/model_executor/models/minicpmo.py index 8278afc017..d4aaeac6ad 100644 --- a/vllm/model_executor/models/minicpmo.py +++ b/vllm/model_executor/models/minicpmo.py @@ -282,28 +282,28 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo): def get_hf_processor(self, **kwargs: object) -> "MiniCPMOProcessor": """Get vendored MiniCPMOProcessor for multimodal (image+audio) inputs. - Creates a vendored processor that reuses the HF image processor, - feature extractor, and tokenizer; applies the correct audio pooling - configuration; and converts numpy arrays in the image processor to - lists for serialization compatibility. The returned processor is + Creates a vendored processor that uses the checkpoint-specific HF image + processor, feature extractor, and tokenizer; applies the correct audio + pooling configuration; and converts numpy arrays in the image processor + to lists for serialization compatibility. The returned processor is compatible with Transformers v5. """ import numpy as np hf_processor = self.ctx.get_hf_processor(**kwargs) + image_processor = self._get_checkpoint_image_processor(**kwargs) from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor # Create vendored processor with correct configuration vendored_processor = MiniCPMOProcessor( - image_processor=hf_processor.image_processor, + image_processor=image_processor, feature_extractor=hf_processor.feature_extractor, tokenizer=hf_processor.tokenizer, pool_step=self.get_default_audio_pool_step(), ) # Convert numpy arrays in image processor to lists for serialization - image_processor = vendored_processor.image_processor for attr in ("mean", "std"): val = getattr(image_processor, attr, None) if val is not None and isinstance(val, np.ndarray): diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index 55853c86e2..d5c2c23e3e 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -611,16 +611,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): def get_hf_config(self): return self.ctx.get_hf_config() - def get_hf_processor(self, **kwargs: object): + def _get_checkpoint_image_processor(self, **kwargs: object): model_config = self.ctx.model_config processor_cls = self._image_processor_cls merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs) - # AutoProcessor only for tokenizer; its image_processor is resolved by - # class name and can pick the wrong checkpoint across MiniCPM-V versions. - hf_processor = self.ctx.get_hf_processor(**kwargs) - - image_processor = cached_get_image_processor( + return cached_get_image_processor( model_config.model, revision=model_config.revision, trust_remote_code=model_config.trust_remote_code, @@ -628,6 +624,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): **merged_kwargs, ) + def get_hf_processor(self, **kwargs: object): + # AutoProcessor only for tokenizer; its image_processor is resolved by + # class name and can pick the wrong MiniCPM checkpoint. + hf_processor = self.ctx.get_hf_processor(**kwargs) + image_processor = self._get_checkpoint_image_processor(**kwargs) + from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor vendored_processor = MiniCPMVProcessor(