[Bugfix][MM] Fix MiniCPM-V placeholder replacement and image processor loading on Transformers v5 (#48413)

Signed-off-by: YunzhuLu <lucia.yunzhu@gmail.com>
Signed-off-by: Yunzhu Lu <lucia.yunzhu@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Yunzhu Lu
2026-08-05 21:12:00 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent 22170354d8
commit f84df12c6a
4 changed files with 218 additions and 10 deletions
@@ -0,0 +1,74 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MiniCPMV's multimodal preprocessing."""
import numpy as np
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
"""Calls with different kwargs must not reuse stale processor instances."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
info = processor.info
processor_1 = info.get_hf_processor(max_slice_nums=1)
processor_2 = info.get_hf_processor(max_slice_nums=2)
assert processor_1.image_processor.max_slice_nums == 1
assert processor_2.image_processor.max_slice_nums == 2
@pytest.mark.parametrize(
"model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")]
)
def test_image_processor_for_dif_model(model_ids):
model_id_25, model_id_4 = model_ids
ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1})
processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config)
image_processor_25 = processor_25.info.get_image_processor()
ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1})
processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config)
image_processor_4 = processor_4.info.get_image_processor()
assert type(image_processor_25) is not type(image_processor_4)
assert type(image_processor_25).__module__ != type(image_processor_4).__module__
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
def test_prompt_has_dif_BPE_boundaries_in_context(model_id: str):
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = ctx.get_tokenizer()
messages = [
{"role": "user", "content": "(<image>./</image>)\nWhat is in this image?"}
]
prompt = tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
)
image = np.zeros((768, 1024, 3), dtype=np.uint8)
mm_items = processor.info.parse_mm_data({"image": [image]})
processed = processor(
prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
image_placeholders = processed["mm_placeholders"].get("image", [])
assert len(image_placeholders) == 1
assert image_placeholders[0].length > 0
-7
View File
@@ -1107,13 +1107,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"4.0": "openbmb/MiniCPM-V-4", "4.0": "openbmb/MiniCPM-V-4",
"4.5": "openbmb/MiniCPM-V-4_5", "4.5": "openbmb/MiniCPM-V-4_5",
}, },
max_transformers_version="4.57",
transformers_version_reason={
"vllm": (
"MiniCPMVBatchFeature is incompatible with its base class in "
"Transformers v5. See https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/discussions/78"
)
},
trust_remote_code=True, trust_remote_code=True,
), ),
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo( "MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
+111 -3
View File
@@ -27,7 +27,7 @@
import math import math
from collections import defaultdict from collections import defaultdict
from collections.abc import Callable, Iterable, Mapping, Sequence, Set from collections.abc import Callable, Iterable, Mapping, Sequence, Set
from functools import partial from functools import cached_property, partial
from itertools import chain from itertools import chain
from typing import Annotated, Any, Literal, TypeAlias from typing import Annotated, Any, Literal, TypeAlias
@@ -37,6 +37,10 @@ import torch.types
from torch import nn from torch import nn
from torch.nn.init import trunc_normal_ from torch.nn.init import trunc_normal_
from transformers import BatchFeature, PretrainedConfig from transformers import BatchFeature, PretrainedConfig
from transformers.dynamic_module_utils import (
get_class_from_dynamic_module,
resolve_trust_remote_code,
)
from typing_extensions import TypeVar from typing_extensions import TypeVar
from vllm.config import VllmConfig from vllm.config import VllmConfig
@@ -74,6 +78,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import ( from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor, BaseMultiModalProcessor,
BaseProcessingInfo, BaseProcessingInfo,
MultiModalPromptUpdates,
PlaceholderFeaturesInfo,
PromptReplacement, PromptReplacement,
PromptUpdate, PromptUpdate,
PromptUpdateDetails, PromptUpdateDetails,
@@ -82,6 +88,11 @@ from vllm.multimodal.processing.processor import (
) )
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import (
_merge_mm_kwargs,
cached_get_image_processor,
)
from vllm.transformers_utils.utils import convert_model_repo_to_path
from vllm.utils.collection_utils import flatten_2d_lists from vllm.utils.collection_utils import flatten_2d_lists
from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import set_default_torch_dtype from vllm.utils.torch_utils import set_default_torch_dtype
@@ -559,16 +570,63 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
image_pattern = "(<image>./</image>)" image_pattern = "(<image>./</image>)"
video_pattern = "(<video>./</video>)" video_pattern = "(<video>./</video>)"
@cached_property
def _image_processor_cls(self):
model_config = self.ctx.model_config
model_path = convert_model_repo_to_path(model_config.model)
from transformers import ImageProcessingMixin
image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict(
model_path,
revision=model_config.revision,
token=model_config.hf_token,
)
auto_map = image_processor_config.get("auto_map") or {}
class_ref = auto_map.get("AutoImageProcessor")
if not class_ref:
raise ValueError(
"Missing auto_map['AutoImageProcessor'] in image processor config "
f"for {model_config.model!r}"
)
resolve_trust_remote_code(
model_config.trust_remote_code,
model_config.model,
has_local_code=False,
has_remote_code=True,
)
return get_class_from_dynamic_module(
class_ref,
model_config.model,
revision=model_config.revision,
)
def get_hf_config(self): def get_hf_config(self):
return self.ctx.get_hf_config() return self.ctx.get_hf_config()
def get_hf_processor(self, **kwargs: object): def get_hf_processor(self, **kwargs: object):
model_config = self.ctx.model_config
processor_cls = self._image_processor_cls
merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs)
# AutoProcessor only for tokenizer; its image_processor is resolved by
# class name and can pick the wrong checkpoint across MiniCPM-V versions.
hf_processor = self.ctx.get_hf_processor(**kwargs) hf_processor = self.ctx.get_hf_processor(**kwargs)
image_processor = cached_get_image_processor(
model_config.model,
revision=model_config.revision,
trust_remote_code=model_config.trust_remote_code,
processor_cls_overrides=processor_cls,
**merged_kwargs,
)
from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
vendored_processor = MiniCPMVProcessor( vendored_processor = MiniCPMVProcessor(
image_processor=hf_processor.image_processor, image_processor=image_processor,
tokenizer=hf_processor.tokenizer, tokenizer=hf_processor.tokenizer,
version=self.get_model_version(), version=self.get_model_version(),
) )
@@ -576,7 +634,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
# NumPy arrays are considered as Iterable but not Sequence in # NumPy arrays are considered as Iterable but not Sequence in
# https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428 # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
image_processor = hf_processor.image_processor # type: ignore
# transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std` # transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std`
for attr in ("mean", "std", "image_mean", "image_std"): for attr in ("mean", "std", "image_mean", "image_std"):
val = getattr(image_processor, attr, None) val = getattr(image_processor, attr, None)
@@ -863,6 +920,57 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
**self.process_videos(mm_data, mm_kwargs, tok_kwargs), **self.process_videos(mm_data, mm_kwargs, tok_kwargs),
} }
def _apply_prompt_updates(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""Apply multi-modal prompt updates to token IDs."""
tokenizer = self.info.get_tokenizer()
new_token_ids, match_result = self._apply_token_matches(
token_ids,
mm_prompt_updates,
)
# If the search text does not represent a special token,
# it may have different token IDs in the prompt, because
# the tokens may go across the boundaries of the search text.
# ----
# e.g. when searching for "foo" in "food", if "food" itself makes
# up a token, then the token ID of "foo" will not appear at all
# ----
# Since it is inefficient to search for all possible tokenizations
# of the search text in the prompt, we instead perform string-based
# updates on the decoded token IDs, then encode them back.
if not all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
new_token_ids, match_result = self._apply_text_matches_as_segmented_tokens(
_seq2text(tokenizer, token_ids, use_cache=False),
mm_prompt_updates,
)
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
for modality, update_idxs in match_result.items():
for item_idx, update_idx in enumerate(update_idxs):
assert update_idx is not None, (
"Failed to apply prompt replacement for "
f"mm_items[{modality!r}][{item_idx}]"
)
matched_updates[modality].append(
[mm_prompt_updates[modality][item_idx][update_idx]]
)
placeholders = self._find_mm_placeholders(
new_token_ids,
dict(matched_updates),
)
return new_token_ids, placeholders
def _base_call_hf_processor( def _base_call_hf_processor(
self, self,
prompts: list[str], prompts: list[str],
+33
View File
@@ -903,6 +903,29 @@ def apply_text_matches(
return "".join(texts), result return "".join(texts), result
def apply_text_matches_as_segmented_tokens(
prompt: str,
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: TokenizerLike | None,
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
"""
Apply the updates in `mm_prompt_updates` to `prompt`.
Matches are exclusive even when multiple modalities share
the same placeholder tokens. In that case, the modality that
appears earlier in `mm_prompt_updates` takes priority.
Each segment is encoded separately instead of being joined into one
string and encoded in a single pass. Joining first would let BPE merge
tokens across a segment boundary, silently change how a text
(non-special-token) placeholder is tokenized.
"""
texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
token_id_seqs = [_seq2tokens(tokenizer, text, use_cache=False) for text in texts]
return flatten_2d_lists(token_id_seqs), result
def _iter_placeholders( def _iter_placeholders(
prompt: list[int], prompt: list[int],
mm_prompt_updates: "MultiModalPromptUpdates", mm_prompt_updates: "MultiModalPromptUpdates",
@@ -1584,6 +1607,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
return apply_text_matches(prompt, mm_prompt_updates, tokenizer) return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
def _apply_text_matches_as_segmented_tokens(
self,
prompt: str,
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
tokenizer = self.info.get_tokenizer()
return apply_text_matches_as_segmented_tokens(
prompt, mm_prompt_updates, tokenizer
)
def _apply_prompt_updates( def _apply_prompt_updates(
self, self,
token_ids: list[int], token_ids: list[int],