[Bugfix][MM] Fix MiniCPM-V placeholder replacement and image processor loading on Transformers v5 (#48413)
Signed-off-by: YunzhuLu <lucia.yunzhu@gmail.com> Signed-off-by: Yunzhu Lu <lucia.yunzhu@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for MiniCPMV's multimodal preprocessing."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
|
||||
def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
|
||||
"""Calls with different kwargs must not reuse stale processor instances."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
info = processor.info
|
||||
processor_1 = info.get_hf_processor(max_slice_nums=1)
|
||||
processor_2 = info.get_hf_processor(max_slice_nums=2)
|
||||
assert processor_1.image_processor.max_slice_nums == 1
|
||||
assert processor_2.image_processor.max_slice_nums == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")]
|
||||
)
|
||||
def test_image_processor_for_dif_model(model_ids):
|
||||
model_id_25, model_id_4 = model_ids
|
||||
|
||||
ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1})
|
||||
processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config)
|
||||
image_processor_25 = processor_25.info.get_image_processor()
|
||||
|
||||
ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1})
|
||||
processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config)
|
||||
image_processor_4 = processor_4.info.get_image_processor()
|
||||
|
||||
assert type(image_processor_25) is not type(image_processor_4)
|
||||
assert type(image_processor_25).__module__ != type(image_processor_4).__module__
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
|
||||
def test_prompt_has_dif_BPE_boundaries_in_context(model_id: str):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
tokenizer = ctx.get_tokenizer()
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "(<image>./</image>)\nWhat is in this image?"}
|
||||
]
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
image = np.zeros((768, 1024, 3), dtype=np.uint8)
|
||||
|
||||
mm_items = processor.info.parse_mm_data({"image": [image]})
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
image_placeholders = processed["mm_placeholders"].get("image", [])
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length > 0
|
||||
Reference in New Issue
Block a user