[Bugfix] Fix O(n²) multimodal string prompt processing (#29667)

Signed-off-by: mertunsall <mertunsal1905@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Mert Unsal
2025-11-28 16:10:39 -08:00
committed by GitHub
co-authored by Roger Wang
parent 6173682b6e
commit c625d7b1c6
2 changed files with 65 additions and 33 deletions
+29 -33
View File
@@ -742,7 +742,6 @@ def _apply_matches(
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: AnyTokenizer,
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
prompt_len = len(prompt)
mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
out_seqs = list[str | list[int]]()
@@ -750,16 +749,15 @@ def _apply_matches(
m: [None] * len(items) for m, items in mm_prompt_updates.items()
}
# Early exit if no items to find
mm_found_counts = {
m: sum(r is not None for r in res) for m, res in out_result.items()
}
if _all_items_found(mm_item_counts, mm_found_counts):
return [prompt], out_result
start_idx = prev_end_idx = 0
while start_idx < max(prompt_len, 1): # Allow inserts into empty prompt
found = False
prev_end_idx = 0
while True:
mode, matches_to_apply = _find_matches(
prompt,
mm_prompt_updates,
@@ -768,39 +766,37 @@ def _apply_matches(
current_result=out_result,
)
if mode is not None:
for (modality, item_idx), (match, update_idx) in matches_to_apply:
found = True
if mode is None:
break # No more matches to find
matched_update = mm_prompt_updates[modality][item_idx][update_idx]
matched_content = matched_update.content.full
for (modality, item_idx), (match, update_idx) in matches_to_apply:
matched_update = mm_prompt_updates[modality][item_idx][update_idx]
matched_content = matched_update.content.full
if mode == UpdateMode.INSERT:
end_idx_to_insert = match.end_idx
elif mode == UpdateMode.REPLACE:
end_idx_to_insert = match.start_idx
else:
assert_never(mode)
if mode == UpdateMode.INSERT:
end_idx_to_insert = match.end_idx
elif mode == UpdateMode.REPLACE:
end_idx_to_insert = match.start_idx
else:
assert_never(mode)
out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
out_seqs.append(
_seq2text(tokenizer, matched_content)
if isinstance(prompt, str)
else _seq2tokens(tokenizer, matched_content)
)
out_result[modality][item_idx] = update_idx
out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
out_seqs.append(
_seq2text(tokenizer, matched_content)
if isinstance(prompt, str)
else _seq2tokens(tokenizer, matched_content)
)
out_result[modality][item_idx] = update_idx
# Exclude overlapping matches
start_idx = prev_end_idx = match.end_idx
# Exclude overlapping matches
prev_end_idx = match.end_idx
mm_found_counts = {
m: sum(r is not None for r in res) for m, res in out_result.items()
}
if _all_items_found(mm_item_counts, mm_found_counts):
break
if not found:
start_idx += 1
# Early exit if all items found
mm_found_counts = {
m: sum(r is not None for r in res) for m, res in out_result.items()
}
if _all_items_found(mm_item_counts, mm_found_counts):
break
out_seqs.append(prompt[prev_end_idx:])