[Core][Multimodal] Skip redundant placeholder scan when token match succeeds (#52925)

Co-authored-by: shenyiqin <shenyiqin1@huawei.com>
This commit is contained in:
Yiqin
2026-08-20 15:04:29 +00:00
committed by GitHub
co-authored by shenyiqin
parent bd8865a299
commit cb09dd7488
4 changed files with 560 additions and 196 deletions
+123 -33
View File
@@ -958,6 +958,64 @@ def apply_token_matches(
return flatten_2d_lists(token_id_seqs), result
def _apply_token_matches_with_placeholders(
token_ids: list[int],
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: TokenizerLike | None,
) -> tuple[
list[int],
"MultiModalPromptUpdatesApplyResult",
Mapping[str, list[PlaceholderFeaturesInfo]],
]:
matched_updates, result = _plan_prompt_updates(
token_ids,
mm_prompt_updates,
tokenizer,
)
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
modality: [] for modality in mm_prompt_updates
}
new_token_ids = list[int]()
prev_end_idx = 0
for matched_update in matched_updates:
update = matched_update.update
match = matched_update.match
matched_content = update.content.full
if update.mode == UpdateMode.INSERT:
end_idx_to_insert = match.end_idx
elif update.mode == UpdateMode.REPLACE:
end_idx_to_insert = match.start_idx
else:
assert_never(update.mode)
new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
start_idx = len(new_token_ids)
tokens = _seq2tokens(tokenizer, matched_content)
if tokens:
content_is_embed = update.content.is_embed
if content_is_embed is not None:
content_is_embed = content_is_embed(tokenizer, matched_content)
placeholders[update.modality].append(
PlaceholderFeaturesInfo(
modality=update.modality,
item_idx=update.item_idx,
start_idx=start_idx,
tokens=tokens,
is_embed=content_is_embed,
)
)
new_token_ids.extend(tokens)
prev_end_idx = match.end_idx
new_token_ids.extend(token_ids[prev_end_idx:])
return new_token_ids, result, placeholders
def apply_text_matches(
prompt: str,
mm_prompt_updates: "MultiModalPromptUpdates",
@@ -1671,6 +1729,22 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
tokenizer = self.info.get_tokenizer()
return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
def _apply_token_matches_with_placeholders(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[
list[int],
MultiModalPromptUpdatesApplyResult,
Mapping[str, list[PlaceholderFeaturesInfo]],
]:
tokenizer = self.info.get_tokenizer()
return _apply_token_matches_with_placeholders(
token_ids,
mm_prompt_updates,
tokenizer,
)
def _apply_text_matches(
self,
prompt: str,
@@ -1689,40 +1763,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt, mm_prompt_updates, tokenizer
)
def _apply_prompt_updates(
def _matched_updates_from_result(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""Apply multi-modal prompt updates to token IDs."""
tokenizer = self.info.get_tokenizer()
new_token_ids, match_result = self._apply_token_matches(
token_ids,
mm_prompt_updates,
)
# If the search text does not represent a special token,
# it may have different token IDs in the prompt, because
# the tokens may go across the boundaries of the search text.
# ----
# e.g. when searching for "foo" in "food", if "food" itself makes
# up a token, then the token ID of "foo" will not appear at all
# ----
# Since it is inefficient to search for all possible tokenizations
# of the search text in the prompt, we instead perform string-based
# updates on the decoded token IDs, then encode them back.
if not all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
new_text, match_result = self._apply_text_matches(
_seq2text(tokenizer, token_ids, use_cache=False),
mm_prompt_updates,
)
new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
match_result: MultiModalPromptUpdatesApplyResult,
) -> dict[str, list[Sequence[ResolvedPromptUpdate]]]:
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
for modality, update_idxs in match_result.items():
for item_idx, update_idx in enumerate(update_idxs):
@@ -1735,9 +1780,54 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
[mm_prompt_updates[modality][item_idx][update_idx]]
)
return dict(matched_updates)
def _apply_prompt_updates(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""Apply multi-modal prompt updates to token IDs."""
tokenizer = self.info.get_tokenizer()
new_token_ids, match_result, placeholders = (
self._apply_token_matches_with_placeholders(
token_ids,
mm_prompt_updates,
)
)
if all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
placeholders = {
modality: modality_placeholders
for modality, modality_placeholders in placeholders.items()
if modality_placeholders
}
return new_token_ids, placeholders
# If the search text does not represent a special token,
# it may have different token IDs in the prompt, because
# the tokens may go across the boundaries of the search text.
# ----
# e.g. when searching for "foo" in "food", if "food" itself makes
# up a token, then the token ID of "foo" will not appear at all
# ----
# Since it is inefficient to search for all possible tokenizations
# of the search text in the prompt, we instead perform string-based
# updates on the decoded token IDs, then encode them back.
new_text, match_result = self._apply_text_matches(
_seq2text(tokenizer, token_ids, use_cache=False),
mm_prompt_updates,
)
new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
placeholders = self._find_mm_placeholders(
new_token_ids,
dict(matched_updates),
self._matched_updates_from_result(mm_prompt_updates, match_result),
)
return new_token_ids, placeholders