[Bugfix][MM] Fix MiniCPM-V placeholder replacement and image processor loading on Transformers v5 (#48413)

Signed-off-by: YunzhuLu <lucia.yunzhu@gmail.com>
Signed-off-by: Yunzhu Lu <lucia.yunzhu@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Yunzhu Lu
2026-08-05 21:12:00 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent 22170354d8
commit f84df12c6a
4 changed files with 218 additions and 10 deletions
+33
View File
@@ -903,6 +903,29 @@ def apply_text_matches(
return "".join(texts), result
def apply_text_matches_as_segmented_tokens(
prompt: str,
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: TokenizerLike | None,
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
"""
Apply the updates in `mm_prompt_updates` to `prompt`.
Matches are exclusive even when multiple modalities share
the same placeholder tokens. In that case, the modality that
appears earlier in `mm_prompt_updates` takes priority.
Each segment is encoded separately instead of being joined into one
string and encoded in a single pass. Joining first would let BPE merge
tokens across a segment boundary, silently change how a text
(non-special-token) placeholder is tokenized.
"""
texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
token_id_seqs = [_seq2tokens(tokenizer, text, use_cache=False) for text in texts]
return flatten_2d_lists(token_id_seqs), result
def _iter_placeholders(
prompt: list[int],
mm_prompt_updates: "MultiModalPromptUpdates",
@@ -1584,6 +1607,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
tokenizer = self.info.get_tokenizer()
return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
def _apply_text_matches_as_segmented_tokens(
self,
prompt: str,
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
tokenizer = self.info.get_tokenizer()
return apply_text_matches_as_segmented_tokens(
prompt, mm_prompt_updates, tokenizer
)
def _apply_prompt_updates(
self,
token_ids: list[int],