[Core][Multimodal] Skip redundant placeholder scan when token match succeeds (#52925)

Co-authored-by: shenyiqin <shenyiqin1@huawei.com>
This commit is contained in:
Yiqin
2026-08-20 15:04:29 +00:00
committed by GitHub
co-authored by shenyiqin
parent bd8865a299
commit cb09dd7488
4 changed files with 560 additions and 196 deletions
+34
View File
@@ -373,6 +373,40 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
return token_ids, res
def _apply_token_matches_with_placeholders(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[
list[int],
MultiModalPromptUpdatesApplyResult,
Mapping[str, list[PlaceholderFeaturesInfo]],
]:
new_token_ids, match_result = self._apply_token_matches(
token_ids,
mm_prompt_updates,
)
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
modality: [] for modality in mm_prompt_updates
}
if all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
placeholders = dict(
self._find_mm_placeholders(
new_token_ids,
self._matched_updates_from_result(
mm_prompt_updates,
match_result,
),
)
)
return new_token_ids, match_result, placeholders
def _find_mm_placeholders(
self,
new_token_ids: list[int],
+34
View File
@@ -391,6 +391,40 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
return token_ids, res
def _apply_token_matches_with_placeholders(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[
list[int],
MultiModalPromptUpdatesApplyResult,
Mapping[str, list[PlaceholderFeaturesInfo]],
]:
new_token_ids, match_result = self._apply_token_matches(
token_ids,
mm_prompt_updates,
)
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
modality: [] for modality in mm_prompt_updates
}
if all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
placeholders = dict(
self._find_mm_placeholders(
new_token_ids,
self._matched_updates_from_result(
mm_prompt_updates,
match_result,
),
)
)
return new_token_ids, match_result, placeholders
def _find_mm_placeholders(
self,
new_token_ids: list[int],