diff --git a/tests/models/multimodal/processing/test_minicpmv.py b/tests/models/multimodal/processing/test_minicpmv.py
new file mode 100644
index 0000000000..9424ea3959
--- /dev/null
+++ b/tests/models/multimodal/processing/test_minicpmv.py
@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""Tests for MiniCPMV's multimodal preprocessing."""
+
+import numpy as np
+import pytest
+
+from vllm.multimodal import MULTIMODAL_REGISTRY
+
+from ...utils import build_model_context
+
+
+@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
+def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
+ """Calls with different kwargs must not reuse stale processor instances."""
+ ctx = build_model_context(
+ model_id,
+ limit_mm_per_prompt={"image": 1},
+ )
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ info = processor.info
+ processor_1 = info.get_hf_processor(max_slice_nums=1)
+ processor_2 = info.get_hf_processor(max_slice_nums=2)
+ assert processor_1.image_processor.max_slice_nums == 1
+ assert processor_2.image_processor.max_slice_nums == 2
+
+
+@pytest.mark.parametrize(
+ "model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")]
+)
+def test_image_processor_for_dif_model(model_ids):
+ model_id_25, model_id_4 = model_ids
+
+ ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1})
+ processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config)
+ image_processor_25 = processor_25.info.get_image_processor()
+
+ ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1})
+ processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config)
+ image_processor_4 = processor_4.info.get_image_processor()
+
+ assert type(image_processor_25) is not type(image_processor_4)
+ assert type(image_processor_25).__module__ != type(image_processor_4).__module__
+
+
+@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
+def test_prompt_has_dif_BPE_boundaries_in_context(model_id: str):
+ ctx = build_model_context(
+ model_id,
+ limit_mm_per_prompt={"image": 1},
+ )
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ tokenizer = ctx.get_tokenizer()
+
+ messages = [
+ {"role": "user", "content": "(./)\nWhat is in this image?"}
+ ]
+ prompt = tokenizer.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True,
+ )
+ image = np.zeros((768, 1024, 3), dtype=np.uint8)
+
+ mm_items = processor.info.parse_mm_data({"image": [image]})
+ processed = processor(
+ prompt,
+ mm_items=mm_items,
+ hf_processor_mm_kwargs={},
+ )
+ image_placeholders = processed["mm_placeholders"].get("image", [])
+ assert len(image_placeholders) == 1
+ assert image_placeholders[0].length > 0
diff --git a/tests/models/registry.py b/tests/models/registry.py
index a37d02299e..e3b615dd7d 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -1107,13 +1107,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"4.0": "openbmb/MiniCPM-V-4",
"4.5": "openbmb/MiniCPM-V-4_5",
},
- max_transformers_version="4.57",
- transformers_version_reason={
- "vllm": (
- "MiniCPMVBatchFeature is incompatible with its base class in "
- "Transformers v5. See https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/discussions/78"
- )
- },
trust_remote_code=True,
),
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py
index 97485d17ed..4a3ad03ed7 100644
--- a/vllm/model_executor/models/minicpmv.py
+++ b/vllm/model_executor/models/minicpmv.py
@@ -27,7 +27,7 @@
import math
from collections import defaultdict
from collections.abc import Callable, Iterable, Mapping, Sequence, Set
-from functools import partial
+from functools import cached_property, partial
from itertools import chain
from typing import Annotated, Any, Literal, TypeAlias
@@ -37,6 +37,10 @@ import torch.types
from torch import nn
from torch.nn.init import trunc_normal_
from transformers import BatchFeature, PretrainedConfig
+from transformers.dynamic_module_utils import (
+ get_class_from_dynamic_module,
+ resolve_trust_remote_code,
+)
from typing_extensions import TypeVar
from vllm.config import VllmConfig
@@ -74,6 +78,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
+ MultiModalPromptUpdates,
+ PlaceholderFeaturesInfo,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
@@ -82,6 +88,11 @@ from vllm.multimodal.processing.processor import (
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
+from vllm.transformers_utils.processor import (
+ _merge_mm_kwargs,
+ cached_get_image_processor,
+)
+from vllm.transformers_utils.utils import convert_model_repo_to_path
from vllm.utils.collection_utils import flatten_2d_lists
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import set_default_torch_dtype
@@ -559,16 +570,63 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
image_pattern = "(./)"
video_pattern = "()"
+ @cached_property
+ def _image_processor_cls(self):
+ model_config = self.ctx.model_config
+ model_path = convert_model_repo_to_path(model_config.model)
+
+ from transformers import ImageProcessingMixin
+
+ image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict(
+ model_path,
+ revision=model_config.revision,
+ token=model_config.hf_token,
+ )
+
+ auto_map = image_processor_config.get("auto_map") or {}
+ class_ref = auto_map.get("AutoImageProcessor")
+ if not class_ref:
+ raise ValueError(
+ "Missing auto_map['AutoImageProcessor'] in image processor config "
+ f"for {model_config.model!r}"
+ )
+
+ resolve_trust_remote_code(
+ model_config.trust_remote_code,
+ model_config.model,
+ has_local_code=False,
+ has_remote_code=True,
+ )
+ return get_class_from_dynamic_module(
+ class_ref,
+ model_config.model,
+ revision=model_config.revision,
+ )
+
def get_hf_config(self):
return self.ctx.get_hf_config()
def get_hf_processor(self, **kwargs: object):
+ model_config = self.ctx.model_config
+ processor_cls = self._image_processor_cls
+ merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs)
+
+ # AutoProcessor only for tokenizer; its image_processor is resolved by
+ # class name and can pick the wrong checkpoint across MiniCPM-V versions.
hf_processor = self.ctx.get_hf_processor(**kwargs)
+ image_processor = cached_get_image_processor(
+ model_config.model,
+ revision=model_config.revision,
+ trust_remote_code=model_config.trust_remote_code,
+ processor_cls_overrides=processor_cls,
+ **merged_kwargs,
+ )
+
from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
vendored_processor = MiniCPMVProcessor(
- image_processor=hf_processor.image_processor,
+ image_processor=image_processor,
tokenizer=hf_processor.tokenizer,
version=self.get_model_version(),
)
@@ -576,7 +634,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
# NumPy arrays are considered as Iterable but not Sequence in
# https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
- image_processor = hf_processor.image_processor # type: ignore
# transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std`
for attr in ("mean", "std", "image_mean", "image_std"):
val = getattr(image_processor, attr, None)
@@ -863,6 +920,57 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
**self.process_videos(mm_data, mm_kwargs, tok_kwargs),
}
+ def _apply_prompt_updates(
+ self,
+ token_ids: list[int],
+ mm_prompt_updates: MultiModalPromptUpdates,
+ ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
+ """Apply multi-modal prompt updates to token IDs."""
+ tokenizer = self.info.get_tokenizer()
+
+ new_token_ids, match_result = self._apply_token_matches(
+ token_ids,
+ mm_prompt_updates,
+ )
+
+ # If the search text does not represent a special token,
+ # it may have different token IDs in the prompt, because
+ # the tokens may go across the boundaries of the search text.
+ # ----
+ # e.g. when searching for "foo" in "food", if "food" itself makes
+ # up a token, then the token ID of "foo" will not appear at all
+ # ----
+ # Since it is inefficient to search for all possible tokenizations
+ # of the search text in the prompt, we instead perform string-based
+ # updates on the decoded token IDs, then encode them back.
+ if not all(
+ all(update_idx is not None for update_idx in update_idxs)
+ for update_idxs in match_result.values()
+ ):
+ new_token_ids, match_result = self._apply_text_matches_as_segmented_tokens(
+ _seq2text(tokenizer, token_ids, use_cache=False),
+ mm_prompt_updates,
+ )
+
+ matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
+ for modality, update_idxs in match_result.items():
+ for item_idx, update_idx in enumerate(update_idxs):
+ assert update_idx is not None, (
+ "Failed to apply prompt replacement for "
+ f"mm_items[{modality!r}][{item_idx}]"
+ )
+
+ matched_updates[modality].append(
+ [mm_prompt_updates[modality][item_idx][update_idx]]
+ )
+
+ placeholders = self._find_mm_placeholders(
+ new_token_ids,
+ dict(matched_updates),
+ )
+
+ return new_token_ids, placeholders
+
def _base_call_hf_processor(
self,
prompts: list[str],
diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py
index 7addd65360..8fafccc3e7 100644
--- a/vllm/multimodal/processing/processor.py
+++ b/vllm/multimodal/processing/processor.py
@@ -903,6 +903,29 @@ def apply_text_matches(
return "".join(texts), result
+def apply_text_matches_as_segmented_tokens(
+ prompt: str,
+ mm_prompt_updates: "MultiModalPromptUpdates",
+ tokenizer: TokenizerLike | None,
+) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
+ """
+ Apply the updates in `mm_prompt_updates` to `prompt`.
+
+ Matches are exclusive even when multiple modalities share
+ the same placeholder tokens. In that case, the modality that
+ appears earlier in `mm_prompt_updates` takes priority.
+
+ Each segment is encoded separately instead of being joined into one
+ string and encoded in a single pass. Joining first would let BPE merge
+ tokens across a segment boundary, silently change how a text
+ (non-special-token) placeholder is tokenized.
+ """
+ texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
+ token_id_seqs = [_seq2tokens(tokenizer, text, use_cache=False) for text in texts]
+
+ return flatten_2d_lists(token_id_seqs), result
+
+
def _iter_placeholders(
prompt: list[int],
mm_prompt_updates: "MultiModalPromptUpdates",
@@ -1584,6 +1607,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
tokenizer = self.info.get_tokenizer()
return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
+ def _apply_text_matches_as_segmented_tokens(
+ self,
+ prompt: str,
+ mm_prompt_updates: MultiModalPromptUpdates,
+ ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
+ tokenizer = self.info.get_tokenizer()
+ return apply_text_matches_as_segmented_tokens(
+ prompt, mm_prompt_updates, tokenizer
+ )
+
def _apply_prompt_updates(
self,
token_ids: list[int],