[Core][Multimodal] Skip redundant placeholder scan when token match succeeds (#52925)
Co-authored-by: shenyiqin <shenyiqin1@huawei.com>
This commit is contained in:
@@ -373,6 +373,40 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
||||
|
||||
return token_ids, res
|
||||
|
||||
def _apply_token_matches_with_placeholders(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[
|
||||
list[int],
|
||||
MultiModalPromptUpdatesApplyResult,
|
||||
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||
]:
|
||||
new_token_ids, match_result = self._apply_token_matches(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
)
|
||||
|
||||
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
|
||||
modality: [] for modality in mm_prompt_updates
|
||||
}
|
||||
|
||||
if all(
|
||||
all(update_idx is not None for update_idx in update_idxs)
|
||||
for update_idxs in match_result.values()
|
||||
):
|
||||
placeholders = dict(
|
||||
self._find_mm_placeholders(
|
||||
new_token_ids,
|
||||
self._matched_updates_from_result(
|
||||
mm_prompt_updates,
|
||||
match_result,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
return new_token_ids, match_result, placeholders
|
||||
|
||||
def _find_mm_placeholders(
|
||||
self,
|
||||
new_token_ids: list[int],
|
||||
|
||||
@@ -391,6 +391,40 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
|
||||
return token_ids, res
|
||||
|
||||
def _apply_token_matches_with_placeholders(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[
|
||||
list[int],
|
||||
MultiModalPromptUpdatesApplyResult,
|
||||
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||
]:
|
||||
new_token_ids, match_result = self._apply_token_matches(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
)
|
||||
|
||||
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
|
||||
modality: [] for modality in mm_prompt_updates
|
||||
}
|
||||
|
||||
if all(
|
||||
all(update_idx is not None for update_idx in update_idxs)
|
||||
for update_idxs in match_result.values()
|
||||
):
|
||||
placeholders = dict(
|
||||
self._find_mm_placeholders(
|
||||
new_token_ids,
|
||||
self._matched_updates_from_result(
|
||||
mm_prompt_updates,
|
||||
match_result,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
return new_token_ids, match_result, placeholders
|
||||
|
||||
def _find_mm_placeholders(
|
||||
self,
|
||||
new_token_ids: list[int],
|
||||
|
||||
Reference in New Issue
Block a user