From 89ac407e3d8bdb34901dc63af71e4b5732de5ca0 Mon Sep 17 00:00:00 2001 From: Hareesh Gali Date: Mon, 3 Aug 2026 11:27:56 +0300 Subject: [PATCH] [Perf] Speed up multimodal placeholder and token-match scanning (#50716) Signed-off-by: Hareesh Co-authored-by: Claude Fable 5 --- tests/multimodal/test_processing.py | 60 ++++++++++ vllm/multimodal/processing/processor.py | 143 ++++++++++++++++-------- 2 files changed, 158 insertions(+), 45 deletions(-) diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 2153cd2c25..72b6344380 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -1097,3 +1097,63 @@ def test_apply_matches_no_match_exits_quickly(): # Should complete in < 100ms (was taking seconds before the fix) assert elapsed < 0.1, f"_apply_matches took {elapsed:.2f}s, expected < 0.1s" assert "".join(result) == long_prompt + + +def test_iter_token_matches_rejects_negative_start_idx(): + with pytest.raises(ValueError, match="non-negative"): + list(iter_token_matches([1, 2, 3], [2], start_idx=-1)) + + +def test_find_mm_placeholders_avoids_quadratic_false_prefixes(): + """ + Test that placeholder scanning stays linear under adversarial candidates. + + The fast-forward scan must not rescan the prompt tail per position when + one candidate's first token never occurs (forcing a full search) while + another's occurs at every position (forcing single-step advances). + """ + prompt = [1] * 30_000 + mm_prompt_updates = { + "absent": [[PromptReplacement("absent", [0], [999, 0]).resolve(0)]], + "frequent_false_prefix": [ + [PromptReplacement("frequent_false_prefix", [0], [1, 2]).resolve(0)] + ], + } + + start = time.perf_counter() + result = find_mm_placeholders(prompt, mm_prompt_updates, tokenizer=None) + elapsed = time.perf_counter() - start + + assert result == {} + assert elapsed < 0.5, f"find_mm_placeholders took {elapsed:.2f}s, expected < 0.5s" + + +@pytest.mark.parametrize( + "prompt", + [ + # Empty prompt: the scan loop is never entered + [], + # Non-empty prompt: the scan runs but never finds the first item, + # so the second item must stay unresolved + [1, 2, 3, 4, 5], + ], +) +def test_find_mm_placeholders_resolves_content_lazily(prompt): + """ + Test that content of items the scan never reaches is not resolved. + + With `tokenizer=None`, resolving string content raises; the scan must + return no placeholders instead of raising on the second item. + """ + result = find_mm_placeholders( + prompt, + { + "image": [ + [PromptReplacement("image", [0], [999]).resolve(0)], + [PromptReplacement("image", [0], "never reached").resolve(1)], + ] + }, + tokenizer=None, + ) + + assert result == {} diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index f43b45ff7e..7addd65360 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -627,13 +627,25 @@ def iter_token_matches( Note that empty matches are ignored. """ + if start_idx < 0: + raise ValueError("start_idx must be non-negative") + prompt_len = len(token_ids) match_len = len(match_ids) if match_len == 0: return - while start_idx < prompt_len - match_len + 1: + first_id = match_ids[0] + last_start_idx = prompt_len - match_len + + while start_idx <= last_start_idx: + # Fast-forward to the next candidate position using a C-level scan + try: + start_idx = token_ids.index(first_id, start_idx, last_start_idx + 1) + except ValueError: + return + end_idx = start_idx + match_len if token_ids[start_idx:end_idx] == match_ids: @@ -708,6 +720,37 @@ def _find_matches( mode: UpdateMode | None = None mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]() + # Items commonly share the same target object (a static PromptUpdate + # target resolves to the same object for every item); scan the prompt once + # per distinct target instead of once per item. The cache is keyed by + # object identity: it implies value equality, costs O(1) per lookup + # (hashing a long target value per lookup would defeat the point), and the + # target objects are kept alive by `mm_prompt_updates` for the duration of + # this call. Items with equal-valued but distinct target objects simply + # miss the cache and scan independently. + # NOTE: ResolvedPromptUpdate.iter_matches depends only on the target, + # prompt, tokenizer and start_idx, so updates sharing a target share the + # same first match within this invocation. Revisit this cache if matching + # ever depends on other update attributes. + first_match_by_target_id = dict[int, PromptTargetMatch | None]() + + def get_first_match(update: "ResolvedPromptUpdate") -> PromptTargetMatch | None: + target = update.target + if isinstance(target, PromptIndex): # Matches are not cacheable + return next( + update.iter_matches(prompt, tokenizer, start_idx=prev_end_idx), + None, + ) + + key = id(target) + if key not in first_match_by_target_id: + first_match_by_target_id[key] = next( + update.iter_matches(prompt, tokenizer, start_idx=prev_end_idx), + None, + ) + + return first_match_by_target_id[key] + for modality, modality_updates in mm_prompt_updates.items(): for item_idx, item_updates in enumerate(modality_updates): if current_result[modality][item_idx] is not None: @@ -717,19 +760,17 @@ def _find_matches( if (modality, item_idx) in mm_matches: break # Already found a match for this item - for match in update.iter_matches( - prompt, - tokenizer, - start_idx=prev_end_idx, - ): - # All matches should share the same mode - if mode is None: - mode = update.mode - elif mode != update.mode: - continue + match = get_first_match(update) + if match is None: + continue - mm_matches[(modality, item_idx)] = match, update_idx - break # Get only the first valid match per item + # All matches should share the same mode + if mode is None: + mode = update.mode + elif mode != update.mode: + continue + + mm_matches[(modality, item_idx)] = match, update_idx # Prioritize earlier matches matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0]) @@ -885,47 +926,59 @@ def _iter_placeholders( prompt_len = len(prompt) start_idx = 0 - while start_idx < prompt_len: - found = False + # The current (unfound) item's updates for each modality, with their + # content resolved to token ids; rebuilt whenever an item is found. + # Items are only resolved once the scan reaches them. + candidates: list[tuple[str, ResolvedPromptUpdate, list[int]]] | None = None - for modality, modality_updates in mm_prompt_updates.items(): - item_idx = item_idx_by_modality[modality] - if item_idx >= mm_item_counts.get(modality, 0): + while start_idx < prompt_len: + if candidates is None: + candidates = [ + (modality, update, _seq2tokens(tokenizer, update.content.full)) + for modality, modality_updates in mm_prompt_updates.items() + if item_idx_by_modality[modality] < mm_item_counts.get(modality, 0) + for update in modality_updates[item_idx_by_modality[modality]] + ] + if not candidates: + return + + found = False + first_token = prompt[start_idx] + + for modality, update, content_tokens_full in candidates: + content_len_full = len(content_tokens_full) + end_idx_full = start_idx + content_len_full + + if content_len_full == 0 or end_idx_full > prompt_len: continue - for update in modality_updates[item_idx]: + # Check the first token before comparing the full slice + if ( + first_token == content_tokens_full[0] + and prompt[start_idx:end_idx_full] == content_tokens_full + ): content = update.content - content_tokens_full = _seq2tokens(tokenizer, content.full) - content_len_full = len(content_tokens_full) - end_idx_full = start_idx + content_len_full + content_is_embed = content.is_embed + if content_is_embed is not None: + content_is_embed = content_is_embed(tokenizer, content.full) - if content_len_full == 0 or end_idx_full > prompt_len: - continue + yield PlaceholderFeaturesInfo( + modality=modality, + item_idx=item_idx_by_modality[modality], + start_idx=start_idx, + tokens=content_tokens_full, + is_embed=content_is_embed, + ) - if prompt[start_idx:end_idx_full] == content_tokens_full: - content_is_embed = content.is_embed - if content_is_embed is not None: - content_is_embed = content_is_embed(tokenizer, content.full) - - yield PlaceholderFeaturesInfo( - modality=modality, - item_idx=item_idx, - start_idx=start_idx, - tokens=content_tokens_full, - is_embed=content_is_embed, - ) - - # Exclude overlapping matches - start_idx = end_idx_full - item_idx_by_modality[modality] += 1 - found = True - break - - if found: + # Exclude overlapping matches + start_idx = end_idx_full + item_idx_by_modality[modality] += 1 if _all_items_found(mm_item_counts, item_idx_by_modality): return - break # Go back to the outer while loop + candidates = None + found = True + break if not found: start_idx += 1