[Core][Multimodal] Skip redundant placeholder scan when token match succeeds (#52925)
Co-authored-by: shenyiqin <shenyiqin1@huawei.com>
This commit is contained in:
@@ -958,6 +958,64 @@ def apply_token_matches(
|
||||
return flatten_2d_lists(token_id_seqs), result
|
||||
|
||||
|
||||
def _apply_token_matches_with_placeholders(
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> tuple[
|
||||
list[int],
|
||||
"MultiModalPromptUpdatesApplyResult",
|
||||
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||
]:
|
||||
matched_updates, result = _plan_prompt_updates(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
tokenizer,
|
||||
)
|
||||
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
|
||||
modality: [] for modality in mm_prompt_updates
|
||||
}
|
||||
|
||||
new_token_ids = list[int]()
|
||||
prev_end_idx = 0
|
||||
for matched_update in matched_updates:
|
||||
update = matched_update.update
|
||||
match = matched_update.match
|
||||
matched_content = update.content.full
|
||||
|
||||
if update.mode == UpdateMode.INSERT:
|
||||
end_idx_to_insert = match.end_idx
|
||||
elif update.mode == UpdateMode.REPLACE:
|
||||
end_idx_to_insert = match.start_idx
|
||||
else:
|
||||
assert_never(update.mode)
|
||||
|
||||
new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
|
||||
start_idx = len(new_token_ids)
|
||||
|
||||
tokens = _seq2tokens(tokenizer, matched_content)
|
||||
if tokens:
|
||||
content_is_embed = update.content.is_embed
|
||||
if content_is_embed is not None:
|
||||
content_is_embed = content_is_embed(tokenizer, matched_content)
|
||||
|
||||
placeholders[update.modality].append(
|
||||
PlaceholderFeaturesInfo(
|
||||
modality=update.modality,
|
||||
item_idx=update.item_idx,
|
||||
start_idx=start_idx,
|
||||
tokens=tokens,
|
||||
is_embed=content_is_embed,
|
||||
)
|
||||
)
|
||||
new_token_ids.extend(tokens)
|
||||
|
||||
prev_end_idx = match.end_idx
|
||||
|
||||
new_token_ids.extend(token_ids[prev_end_idx:])
|
||||
return new_token_ids, result, placeholders
|
||||
|
||||
|
||||
def apply_text_matches(
|
||||
prompt: str,
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
@@ -1671,6 +1729,22 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
|
||||
def _apply_token_matches_with_placeholders(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[
|
||||
list[int],
|
||||
MultiModalPromptUpdatesApplyResult,
|
||||
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||
]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
return _apply_token_matches_with_placeholders(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
tokenizer,
|
||||
)
|
||||
|
||||
def _apply_text_matches(
|
||||
self,
|
||||
prompt: str,
|
||||
@@ -1689,40 +1763,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
prompt, mm_prompt_updates, tokenizer
|
||||
)
|
||||
|
||||
def _apply_prompt_updates(
|
||||
def _matched_updates_from_result(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
"""Apply multi-modal prompt updates to token IDs."""
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
new_token_ids, match_result = self._apply_token_matches(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
)
|
||||
|
||||
# If the search text does not represent a special token,
|
||||
# it may have different token IDs in the prompt, because
|
||||
# the tokens may go across the boundaries of the search text.
|
||||
# ----
|
||||
# e.g. when searching for "foo" in "food", if "food" itself makes
|
||||
# up a token, then the token ID of "foo" will not appear at all
|
||||
# ----
|
||||
# Since it is inefficient to search for all possible tokenizations
|
||||
# of the search text in the prompt, we instead perform string-based
|
||||
# updates on the decoded token IDs, then encode them back.
|
||||
if not all(
|
||||
all(update_idx is not None for update_idx in update_idxs)
|
||||
for update_idxs in match_result.values()
|
||||
):
|
||||
new_text, match_result = self._apply_text_matches(
|
||||
_seq2text(tokenizer, token_ids, use_cache=False),
|
||||
mm_prompt_updates,
|
||||
)
|
||||
|
||||
new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
|
||||
|
||||
match_result: MultiModalPromptUpdatesApplyResult,
|
||||
) -> dict[str, list[Sequence[ResolvedPromptUpdate]]]:
|
||||
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
|
||||
for modality, update_idxs in match_result.items():
|
||||
for item_idx, update_idx in enumerate(update_idxs):
|
||||
@@ -1735,9 +1780,54 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
[mm_prompt_updates[modality][item_idx][update_idx]]
|
||||
)
|
||||
|
||||
return dict(matched_updates)
|
||||
|
||||
def _apply_prompt_updates(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
"""Apply multi-modal prompt updates to token IDs."""
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
new_token_ids, match_result, placeholders = (
|
||||
self._apply_token_matches_with_placeholders(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
)
|
||||
)
|
||||
|
||||
if all(
|
||||
all(update_idx is not None for update_idx in update_idxs)
|
||||
for update_idxs in match_result.values()
|
||||
):
|
||||
placeholders = {
|
||||
modality: modality_placeholders
|
||||
for modality, modality_placeholders in placeholders.items()
|
||||
if modality_placeholders
|
||||
}
|
||||
return new_token_ids, placeholders
|
||||
|
||||
# If the search text does not represent a special token,
|
||||
# it may have different token IDs in the prompt, because
|
||||
# the tokens may go across the boundaries of the search text.
|
||||
# ----
|
||||
# e.g. when searching for "foo" in "food", if "food" itself makes
|
||||
# up a token, then the token ID of "foo" will not appear at all
|
||||
# ----
|
||||
# Since it is inefficient to search for all possible tokenizations
|
||||
# of the search text in the prompt, we instead perform string-based
|
||||
# updates on the decoded token IDs, then encode them back.
|
||||
new_text, match_result = self._apply_text_matches(
|
||||
_seq2text(tokenizer, token_ids, use_cache=False),
|
||||
mm_prompt_updates,
|
||||
)
|
||||
|
||||
new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
|
||||
|
||||
placeholders = self._find_mm_placeholders(
|
||||
new_token_ids,
|
||||
dict(matched_updates),
|
||||
self._matched_updates_from_result(mm_prompt_updates, match_result),
|
||||
)
|
||||
|
||||
return new_token_ids, placeholders
|
||||
|
||||
Reference in New Issue
Block a user