[Bugfix][MM] Fix MiniCPM-o image processor reuse on Transformers v5 (#54501)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
This commit is contained in:
@@ -7,6 +7,7 @@ import numpy as np
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
|
from vllm.multimodal.parse import ImageSize
|
||||||
|
|
||||||
from ...utils import build_model_context
|
from ...utils import build_model_context
|
||||||
|
|
||||||
@@ -27,21 +28,36 @@ def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")]
|
"model_ids",
|
||||||
|
[
|
||||||
|
("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4"),
|
||||||
|
("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-o-2_6"),
|
||||||
|
],
|
||||||
)
|
)
|
||||||
def test_image_processor_for_dif_model(model_ids):
|
def test_image_processor_for_different_models(model_ids):
|
||||||
model_id_25, model_id_4 = model_ids
|
first_model_id, second_model_id = model_ids
|
||||||
|
|
||||||
ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1})
|
first_ctx = build_model_context(
|
||||||
processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config)
|
first_model_id,
|
||||||
image_processor_25 = processor_25.info.get_image_processor()
|
limit_mm_per_prompt={"image": 1},
|
||||||
|
)
|
||||||
|
first_processor = MULTIMODAL_REGISTRY.create_processor(first_ctx.model_config)
|
||||||
|
first_image_processor = first_processor.info.get_image_processor()
|
||||||
|
|
||||||
ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1})
|
second_ctx = build_model_context(
|
||||||
processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config)
|
second_model_id,
|
||||||
image_processor_4 = processor_4.info.get_image_processor()
|
limit_mm_per_prompt={"image": 1},
|
||||||
|
)
|
||||||
|
second_processor = MULTIMODAL_REGISTRY.create_processor(second_ctx.model_config)
|
||||||
|
second_image_processor = second_processor.info.get_image_processor()
|
||||||
|
|
||||||
assert type(image_processor_25) is not type(image_processor_4)
|
second_processor.info.get_sliced_grid(ImageSize(width=128, height=128))
|
||||||
assert type(image_processor_25).__module__ != type(image_processor_4).__module__
|
|
||||||
|
assert type(first_image_processor) is not type(second_image_processor)
|
||||||
|
assert (
|
||||||
|
type(first_image_processor).__module__
|
||||||
|
!= type(second_image_processor).__module__
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
|
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
|
||||||
|
|||||||
@@ -282,28 +282,28 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
|
|||||||
def get_hf_processor(self, **kwargs: object) -> "MiniCPMOProcessor":
|
def get_hf_processor(self, **kwargs: object) -> "MiniCPMOProcessor":
|
||||||
"""Get vendored MiniCPMOProcessor for multimodal (image+audio) inputs.
|
"""Get vendored MiniCPMOProcessor for multimodal (image+audio) inputs.
|
||||||
|
|
||||||
Creates a vendored processor that reuses the HF image processor,
|
Creates a vendored processor that uses the checkpoint-specific HF image
|
||||||
feature extractor, and tokenizer; applies the correct audio pooling
|
processor, feature extractor, and tokenizer; applies the correct audio
|
||||||
configuration; and converts numpy arrays in the image processor to
|
pooling configuration; and converts numpy arrays in the image processor
|
||||||
lists for serialization compatibility. The returned processor is
|
to lists for serialization compatibility. The returned processor is
|
||||||
compatible with Transformers v5.
|
compatible with Transformers v5.
|
||||||
"""
|
"""
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
hf_processor = self.ctx.get_hf_processor(**kwargs)
|
hf_processor = self.ctx.get_hf_processor(**kwargs)
|
||||||
|
image_processor = self._get_checkpoint_image_processor(**kwargs)
|
||||||
|
|
||||||
from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor
|
from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor
|
||||||
|
|
||||||
# Create vendored processor with correct configuration
|
# Create vendored processor with correct configuration
|
||||||
vendored_processor = MiniCPMOProcessor(
|
vendored_processor = MiniCPMOProcessor(
|
||||||
image_processor=hf_processor.image_processor,
|
image_processor=image_processor,
|
||||||
feature_extractor=hf_processor.feature_extractor,
|
feature_extractor=hf_processor.feature_extractor,
|
||||||
tokenizer=hf_processor.tokenizer,
|
tokenizer=hf_processor.tokenizer,
|
||||||
pool_step=self.get_default_audio_pool_step(),
|
pool_step=self.get_default_audio_pool_step(),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Convert numpy arrays in image processor to lists for serialization
|
# Convert numpy arrays in image processor to lists for serialization
|
||||||
image_processor = vendored_processor.image_processor
|
|
||||||
for attr in ("mean", "std"):
|
for attr in ("mean", "std"):
|
||||||
val = getattr(image_processor, attr, None)
|
val = getattr(image_processor, attr, None)
|
||||||
if val is not None and isinstance(val, np.ndarray):
|
if val is not None and isinstance(val, np.ndarray):
|
||||||
|
|||||||
@@ -611,16 +611,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
|||||||
def get_hf_config(self):
|
def get_hf_config(self):
|
||||||
return self.ctx.get_hf_config()
|
return self.ctx.get_hf_config()
|
||||||
|
|
||||||
def get_hf_processor(self, **kwargs: object):
|
def _get_checkpoint_image_processor(self, **kwargs: object):
|
||||||
model_config = self.ctx.model_config
|
model_config = self.ctx.model_config
|
||||||
processor_cls = self._image_processor_cls
|
processor_cls = self._image_processor_cls
|
||||||
merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs)
|
merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs)
|
||||||
|
|
||||||
# AutoProcessor only for tokenizer; its image_processor is resolved by
|
return cached_get_image_processor(
|
||||||
# class name and can pick the wrong checkpoint across MiniCPM-V versions.
|
|
||||||
hf_processor = self.ctx.get_hf_processor(**kwargs)
|
|
||||||
|
|
||||||
image_processor = cached_get_image_processor(
|
|
||||||
model_config.model,
|
model_config.model,
|
||||||
revision=model_config.revision,
|
revision=model_config.revision,
|
||||||
trust_remote_code=model_config.trust_remote_code,
|
trust_remote_code=model_config.trust_remote_code,
|
||||||
@@ -628,6 +624,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
|||||||
**merged_kwargs,
|
**merged_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def get_hf_processor(self, **kwargs: object):
|
||||||
|
# AutoProcessor only for tokenizer; its image_processor is resolved by
|
||||||
|
# class name and can pick the wrong MiniCPM checkpoint.
|
||||||
|
hf_processor = self.ctx.get_hf_processor(**kwargs)
|
||||||
|
image_processor = self._get_checkpoint_image_processor(**kwargs)
|
||||||
|
|
||||||
from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
|
from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
|
||||||
|
|
||||||
vendored_processor = MiniCPMVProcessor(
|
vendored_processor = MiniCPMVProcessor(
|
||||||
|
|||||||
Reference in New Issue
Block a user