[Bugfix][MM] Fix MiniCPM-V placeholder replacement and image processor loading on Transformers v5 (#48413)
Signed-off-by: YunzhuLu <lucia.yunzhu@gmail.com> Signed-off-by: Yunzhu Lu <lucia.yunzhu@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -0,0 +1,74 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
"""Tests for MiniCPMV's multimodal preprocessing."""
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
|
|
||||||
|
from ...utils import build_model_context
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
|
||||||
|
def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
|
||||||
|
"""Calls with different kwargs must not reuse stale processor instances."""
|
||||||
|
ctx = build_model_context(
|
||||||
|
model_id,
|
||||||
|
limit_mm_per_prompt={"image": 1},
|
||||||
|
)
|
||||||
|
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||||
|
info = processor.info
|
||||||
|
processor_1 = info.get_hf_processor(max_slice_nums=1)
|
||||||
|
processor_2 = info.get_hf_processor(max_slice_nums=2)
|
||||||
|
assert processor_1.image_processor.max_slice_nums == 1
|
||||||
|
assert processor_2.image_processor.max_slice_nums == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")]
|
||||||
|
)
|
||||||
|
def test_image_processor_for_dif_model(model_ids):
|
||||||
|
model_id_25, model_id_4 = model_ids
|
||||||
|
|
||||||
|
ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1})
|
||||||
|
processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config)
|
||||||
|
image_processor_25 = processor_25.info.get_image_processor()
|
||||||
|
|
||||||
|
ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1})
|
||||||
|
processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config)
|
||||||
|
image_processor_4 = processor_4.info.get_image_processor()
|
||||||
|
|
||||||
|
assert type(image_processor_25) is not type(image_processor_4)
|
||||||
|
assert type(image_processor_25).__module__ != type(image_processor_4).__module__
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
|
||||||
|
def test_prompt_has_dif_BPE_boundaries_in_context(model_id: str):
|
||||||
|
ctx = build_model_context(
|
||||||
|
model_id,
|
||||||
|
limit_mm_per_prompt={"image": 1},
|
||||||
|
)
|
||||||
|
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||||
|
tokenizer = ctx.get_tokenizer()
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
{"role": "user", "content": "(<image>./</image>)\nWhat is in this image?"}
|
||||||
|
]
|
||||||
|
prompt = tokenizer.apply_chat_template(
|
||||||
|
messages,
|
||||||
|
tokenize=False,
|
||||||
|
add_generation_prompt=True,
|
||||||
|
)
|
||||||
|
image = np.zeros((768, 1024, 3), dtype=np.uint8)
|
||||||
|
|
||||||
|
mm_items = processor.info.parse_mm_data({"image": [image]})
|
||||||
|
processed = processor(
|
||||||
|
prompt,
|
||||||
|
mm_items=mm_items,
|
||||||
|
hf_processor_mm_kwargs={},
|
||||||
|
)
|
||||||
|
image_placeholders = processed["mm_placeholders"].get("image", [])
|
||||||
|
assert len(image_placeholders) == 1
|
||||||
|
assert image_placeholders[0].length > 0
|
||||||
@@ -1107,13 +1107,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
|||||||
"4.0": "openbmb/MiniCPM-V-4",
|
"4.0": "openbmb/MiniCPM-V-4",
|
||||||
"4.5": "openbmb/MiniCPM-V-4_5",
|
"4.5": "openbmb/MiniCPM-V-4_5",
|
||||||
},
|
},
|
||||||
max_transformers_version="4.57",
|
|
||||||
transformers_version_reason={
|
|
||||||
"vllm": (
|
|
||||||
"MiniCPMVBatchFeature is incompatible with its base class in "
|
|
||||||
"Transformers v5. See https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/discussions/78"
|
|
||||||
)
|
|
||||||
},
|
|
||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
),
|
),
|
||||||
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
|
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
|
||||||
|
|||||||
@@ -27,7 +27,7 @@
|
|||||||
import math
|
import math
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from collections.abc import Callable, Iterable, Mapping, Sequence, Set
|
from collections.abc import Callable, Iterable, Mapping, Sequence, Set
|
||||||
from functools import partial
|
from functools import cached_property, partial
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
from typing import Annotated, Any, Literal, TypeAlias
|
from typing import Annotated, Any, Literal, TypeAlias
|
||||||
|
|
||||||
@@ -37,6 +37,10 @@ import torch.types
|
|||||||
from torch import nn
|
from torch import nn
|
||||||
from torch.nn.init import trunc_normal_
|
from torch.nn.init import trunc_normal_
|
||||||
from transformers import BatchFeature, PretrainedConfig
|
from transformers import BatchFeature, PretrainedConfig
|
||||||
|
from transformers.dynamic_module_utils import (
|
||||||
|
get_class_from_dynamic_module,
|
||||||
|
resolve_trust_remote_code,
|
||||||
|
)
|
||||||
from typing_extensions import TypeVar
|
from typing_extensions import TypeVar
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
@@ -74,6 +78,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
|
|||||||
from vllm.multimodal.processing.processor import (
|
from vllm.multimodal.processing.processor import (
|
||||||
BaseMultiModalProcessor,
|
BaseMultiModalProcessor,
|
||||||
BaseProcessingInfo,
|
BaseProcessingInfo,
|
||||||
|
MultiModalPromptUpdates,
|
||||||
|
PlaceholderFeaturesInfo,
|
||||||
PromptReplacement,
|
PromptReplacement,
|
||||||
PromptUpdate,
|
PromptUpdate,
|
||||||
PromptUpdateDetails,
|
PromptUpdateDetails,
|
||||||
@@ -82,6 +88,11 @@ from vllm.multimodal.processing.processor import (
|
|||||||
)
|
)
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
|
from vllm.transformers_utils.processor import (
|
||||||
|
_merge_mm_kwargs,
|
||||||
|
cached_get_image_processor,
|
||||||
|
)
|
||||||
|
from vllm.transformers_utils.utils import convert_model_repo_to_path
|
||||||
from vllm.utils.collection_utils import flatten_2d_lists
|
from vllm.utils.collection_utils import flatten_2d_lists
|
||||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||||
@@ -559,16 +570,63 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
|||||||
image_pattern = "(<image>./</image>)"
|
image_pattern = "(<image>./</image>)"
|
||||||
video_pattern = "(<video>./</video>)"
|
video_pattern = "(<video>./</video>)"
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def _image_processor_cls(self):
|
||||||
|
model_config = self.ctx.model_config
|
||||||
|
model_path = convert_model_repo_to_path(model_config.model)
|
||||||
|
|
||||||
|
from transformers import ImageProcessingMixin
|
||||||
|
|
||||||
|
image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict(
|
||||||
|
model_path,
|
||||||
|
revision=model_config.revision,
|
||||||
|
token=model_config.hf_token,
|
||||||
|
)
|
||||||
|
|
||||||
|
auto_map = image_processor_config.get("auto_map") or {}
|
||||||
|
class_ref = auto_map.get("AutoImageProcessor")
|
||||||
|
if not class_ref:
|
||||||
|
raise ValueError(
|
||||||
|
"Missing auto_map['AutoImageProcessor'] in image processor config "
|
||||||
|
f"for {model_config.model!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
resolve_trust_remote_code(
|
||||||
|
model_config.trust_remote_code,
|
||||||
|
model_config.model,
|
||||||
|
has_local_code=False,
|
||||||
|
has_remote_code=True,
|
||||||
|
)
|
||||||
|
return get_class_from_dynamic_module(
|
||||||
|
class_ref,
|
||||||
|
model_config.model,
|
||||||
|
revision=model_config.revision,
|
||||||
|
)
|
||||||
|
|
||||||
def get_hf_config(self):
|
def get_hf_config(self):
|
||||||
return self.ctx.get_hf_config()
|
return self.ctx.get_hf_config()
|
||||||
|
|
||||||
def get_hf_processor(self, **kwargs: object):
|
def get_hf_processor(self, **kwargs: object):
|
||||||
|
model_config = self.ctx.model_config
|
||||||
|
processor_cls = self._image_processor_cls
|
||||||
|
merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs)
|
||||||
|
|
||||||
|
# AutoProcessor only for tokenizer; its image_processor is resolved by
|
||||||
|
# class name and can pick the wrong checkpoint across MiniCPM-V versions.
|
||||||
hf_processor = self.ctx.get_hf_processor(**kwargs)
|
hf_processor = self.ctx.get_hf_processor(**kwargs)
|
||||||
|
|
||||||
|
image_processor = cached_get_image_processor(
|
||||||
|
model_config.model,
|
||||||
|
revision=model_config.revision,
|
||||||
|
trust_remote_code=model_config.trust_remote_code,
|
||||||
|
processor_cls_overrides=processor_cls,
|
||||||
|
**merged_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
|
from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
|
||||||
|
|
||||||
vendored_processor = MiniCPMVProcessor(
|
vendored_processor = MiniCPMVProcessor(
|
||||||
image_processor=hf_processor.image_processor,
|
image_processor=image_processor,
|
||||||
tokenizer=hf_processor.tokenizer,
|
tokenizer=hf_processor.tokenizer,
|
||||||
version=self.get_model_version(),
|
version=self.get_model_version(),
|
||||||
)
|
)
|
||||||
@@ -576,7 +634,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
|||||||
|
|
||||||
# NumPy arrays are considered as Iterable but not Sequence in
|
# NumPy arrays are considered as Iterable but not Sequence in
|
||||||
# https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
|
# https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
|
||||||
image_processor = hf_processor.image_processor # type: ignore
|
|
||||||
# transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std`
|
# transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std`
|
||||||
for attr in ("mean", "std", "image_mean", "image_std"):
|
for attr in ("mean", "std", "image_mean", "image_std"):
|
||||||
val = getattr(image_processor, attr, None)
|
val = getattr(image_processor, attr, None)
|
||||||
@@ -863,6 +920,57 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
|||||||
**self.process_videos(mm_data, mm_kwargs, tok_kwargs),
|
**self.process_videos(mm_data, mm_kwargs, tok_kwargs),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _apply_prompt_updates(
|
||||||
|
self,
|
||||||
|
token_ids: list[int],
|
||||||
|
mm_prompt_updates: MultiModalPromptUpdates,
|
||||||
|
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||||
|
"""Apply multi-modal prompt updates to token IDs."""
|
||||||
|
tokenizer = self.info.get_tokenizer()
|
||||||
|
|
||||||
|
new_token_ids, match_result = self._apply_token_matches(
|
||||||
|
token_ids,
|
||||||
|
mm_prompt_updates,
|
||||||
|
)
|
||||||
|
|
||||||
|
# If the search text does not represent a special token,
|
||||||
|
# it may have different token IDs in the prompt, because
|
||||||
|
# the tokens may go across the boundaries of the search text.
|
||||||
|
# ----
|
||||||
|
# e.g. when searching for "foo" in "food", if "food" itself makes
|
||||||
|
# up a token, then the token ID of "foo" will not appear at all
|
||||||
|
# ----
|
||||||
|
# Since it is inefficient to search for all possible tokenizations
|
||||||
|
# of the search text in the prompt, we instead perform string-based
|
||||||
|
# updates on the decoded token IDs, then encode them back.
|
||||||
|
if not all(
|
||||||
|
all(update_idx is not None for update_idx in update_idxs)
|
||||||
|
for update_idxs in match_result.values()
|
||||||
|
):
|
||||||
|
new_token_ids, match_result = self._apply_text_matches_as_segmented_tokens(
|
||||||
|
_seq2text(tokenizer, token_ids, use_cache=False),
|
||||||
|
mm_prompt_updates,
|
||||||
|
)
|
||||||
|
|
||||||
|
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
|
||||||
|
for modality, update_idxs in match_result.items():
|
||||||
|
for item_idx, update_idx in enumerate(update_idxs):
|
||||||
|
assert update_idx is not None, (
|
||||||
|
"Failed to apply prompt replacement for "
|
||||||
|
f"mm_items[{modality!r}][{item_idx}]"
|
||||||
|
)
|
||||||
|
|
||||||
|
matched_updates[modality].append(
|
||||||
|
[mm_prompt_updates[modality][item_idx][update_idx]]
|
||||||
|
)
|
||||||
|
|
||||||
|
placeholders = self._find_mm_placeholders(
|
||||||
|
new_token_ids,
|
||||||
|
dict(matched_updates),
|
||||||
|
)
|
||||||
|
|
||||||
|
return new_token_ids, placeholders
|
||||||
|
|
||||||
def _base_call_hf_processor(
|
def _base_call_hf_processor(
|
||||||
self,
|
self,
|
||||||
prompts: list[str],
|
prompts: list[str],
|
||||||
|
|||||||
@@ -903,6 +903,29 @@ def apply_text_matches(
|
|||||||
return "".join(texts), result
|
return "".join(texts), result
|
||||||
|
|
||||||
|
|
||||||
|
def apply_text_matches_as_segmented_tokens(
|
||||||
|
prompt: str,
|
||||||
|
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||||
|
tokenizer: TokenizerLike | None,
|
||||||
|
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
|
||||||
|
"""
|
||||||
|
Apply the updates in `mm_prompt_updates` to `prompt`.
|
||||||
|
|
||||||
|
Matches are exclusive even when multiple modalities share
|
||||||
|
the same placeholder tokens. In that case, the modality that
|
||||||
|
appears earlier in `mm_prompt_updates` takes priority.
|
||||||
|
|
||||||
|
Each segment is encoded separately instead of being joined into one
|
||||||
|
string and encoded in a single pass. Joining first would let BPE merge
|
||||||
|
tokens across a segment boundary, silently change how a text
|
||||||
|
(non-special-token) placeholder is tokenized.
|
||||||
|
"""
|
||||||
|
texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
|
||||||
|
token_id_seqs = [_seq2tokens(tokenizer, text, use_cache=False) for text in texts]
|
||||||
|
|
||||||
|
return flatten_2d_lists(token_id_seqs), result
|
||||||
|
|
||||||
|
|
||||||
def _iter_placeholders(
|
def _iter_placeholders(
|
||||||
prompt: list[int],
|
prompt: list[int],
|
||||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||||
@@ -1584,6 +1607,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
|
return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
|
||||||
|
|
||||||
|
def _apply_text_matches_as_segmented_tokens(
|
||||||
|
self,
|
||||||
|
prompt: str,
|
||||||
|
mm_prompt_updates: MultiModalPromptUpdates,
|
||||||
|
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
|
||||||
|
tokenizer = self.info.get_tokenizer()
|
||||||
|
return apply_text_matches_as_segmented_tokens(
|
||||||
|
prompt, mm_prompt_updates, tokenizer
|
||||||
|
)
|
||||||
|
|
||||||
def _apply_prompt_updates(
|
def _apply_prompt_updates(
|
||||||
self,
|
self,
|
||||||
token_ids: list[int],
|
token_ids: list[int],
|
||||||
|
|||||||
Reference in New Issue
Block a user