[Perf] Speed up multimodal placeholder and token-match scanning (#50716)
Signed-off-by: Hareesh <hgsata@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
d83eb0b36b
commit
89ac407e3d
@@ -1097,3 +1097,63 @@ def test_apply_matches_no_match_exits_quickly():
|
||||
# Should complete in < 100ms (was taking seconds before the fix)
|
||||
assert elapsed < 0.1, f"_apply_matches took {elapsed:.2f}s, expected < 0.1s"
|
||||
assert "".join(result) == long_prompt
|
||||
|
||||
|
||||
def test_iter_token_matches_rejects_negative_start_idx():
|
||||
with pytest.raises(ValueError, match="non-negative"):
|
||||
list(iter_token_matches([1, 2, 3], [2], start_idx=-1))
|
||||
|
||||
|
||||
def test_find_mm_placeholders_avoids_quadratic_false_prefixes():
|
||||
"""
|
||||
Test that placeholder scanning stays linear under adversarial candidates.
|
||||
|
||||
The fast-forward scan must not rescan the prompt tail per position when
|
||||
one candidate's first token never occurs (forcing a full search) while
|
||||
another's occurs at every position (forcing single-step advances).
|
||||
"""
|
||||
prompt = [1] * 30_000
|
||||
mm_prompt_updates = {
|
||||
"absent": [[PromptReplacement("absent", [0], [999, 0]).resolve(0)]],
|
||||
"frequent_false_prefix": [
|
||||
[PromptReplacement("frequent_false_prefix", [0], [1, 2]).resolve(0)]
|
||||
],
|
||||
}
|
||||
|
||||
start = time.perf_counter()
|
||||
result = find_mm_placeholders(prompt, mm_prompt_updates, tokenizer=None)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
assert result == {}
|
||||
assert elapsed < 0.5, f"find_mm_placeholders took {elapsed:.2f}s, expected < 0.5s"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prompt",
|
||||
[
|
||||
# Empty prompt: the scan loop is never entered
|
||||
[],
|
||||
# Non-empty prompt: the scan runs but never finds the first item,
|
||||
# so the second item must stay unresolved
|
||||
[1, 2, 3, 4, 5],
|
||||
],
|
||||
)
|
||||
def test_find_mm_placeholders_resolves_content_lazily(prompt):
|
||||
"""
|
||||
Test that content of items the scan never reaches is not resolved.
|
||||
|
||||
With `tokenizer=None`, resolving string content raises; the scan must
|
||||
return no placeholders instead of raising on the second item.
|
||||
"""
|
||||
result = find_mm_placeholders(
|
||||
prompt,
|
||||
{
|
||||
"image": [
|
||||
[PromptReplacement("image", [0], [999]).resolve(0)],
|
||||
[PromptReplacement("image", [0], "never reached").resolve(1)],
|
||||
]
|
||||
},
|
||||
tokenizer=None,
|
||||
)
|
||||
|
||||
assert result == {}
|
||||
|
||||
@@ -627,13 +627,25 @@ def iter_token_matches(
|
||||
|
||||
Note that empty matches are ignored.
|
||||
"""
|
||||
if start_idx < 0:
|
||||
raise ValueError("start_idx must be non-negative")
|
||||
|
||||
prompt_len = len(token_ids)
|
||||
match_len = len(match_ids)
|
||||
|
||||
if match_len == 0:
|
||||
return
|
||||
|
||||
while start_idx < prompt_len - match_len + 1:
|
||||
first_id = match_ids[0]
|
||||
last_start_idx = prompt_len - match_len
|
||||
|
||||
while start_idx <= last_start_idx:
|
||||
# Fast-forward to the next candidate position using a C-level scan
|
||||
try:
|
||||
start_idx = token_ids.index(first_id, start_idx, last_start_idx + 1)
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
end_idx = start_idx + match_len
|
||||
|
||||
if token_ids[start_idx:end_idx] == match_ids:
|
||||
@@ -708,6 +720,37 @@ def _find_matches(
|
||||
mode: UpdateMode | None = None
|
||||
mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()
|
||||
|
||||
# Items commonly share the same target object (a static PromptUpdate
|
||||
# target resolves to the same object for every item); scan the prompt once
|
||||
# per distinct target instead of once per item. The cache is keyed by
|
||||
# object identity: it implies value equality, costs O(1) per lookup
|
||||
# (hashing a long target value per lookup would defeat the point), and the
|
||||
# target objects are kept alive by `mm_prompt_updates` for the duration of
|
||||
# this call. Items with equal-valued but distinct target objects simply
|
||||
# miss the cache and scan independently.
|
||||
# NOTE: ResolvedPromptUpdate.iter_matches depends only on the target,
|
||||
# prompt, tokenizer and start_idx, so updates sharing a target share the
|
||||
# same first match within this invocation. Revisit this cache if matching
|
||||
# ever depends on other update attributes.
|
||||
first_match_by_target_id = dict[int, PromptTargetMatch | None]()
|
||||
|
||||
def get_first_match(update: "ResolvedPromptUpdate") -> PromptTargetMatch | None:
|
||||
target = update.target
|
||||
if isinstance(target, PromptIndex): # Matches are not cacheable
|
||||
return next(
|
||||
update.iter_matches(prompt, tokenizer, start_idx=prev_end_idx),
|
||||
None,
|
||||
)
|
||||
|
||||
key = id(target)
|
||||
if key not in first_match_by_target_id:
|
||||
first_match_by_target_id[key] = next(
|
||||
update.iter_matches(prompt, tokenizer, start_idx=prev_end_idx),
|
||||
None,
|
||||
)
|
||||
|
||||
return first_match_by_target_id[key]
|
||||
|
||||
for modality, modality_updates in mm_prompt_updates.items():
|
||||
for item_idx, item_updates in enumerate(modality_updates):
|
||||
if current_result[modality][item_idx] is not None:
|
||||
@@ -717,19 +760,17 @@ def _find_matches(
|
||||
if (modality, item_idx) in mm_matches:
|
||||
break # Already found a match for this item
|
||||
|
||||
for match in update.iter_matches(
|
||||
prompt,
|
||||
tokenizer,
|
||||
start_idx=prev_end_idx,
|
||||
):
|
||||
# All matches should share the same mode
|
||||
if mode is None:
|
||||
mode = update.mode
|
||||
elif mode != update.mode:
|
||||
continue
|
||||
match = get_first_match(update)
|
||||
if match is None:
|
||||
continue
|
||||
|
||||
mm_matches[(modality, item_idx)] = match, update_idx
|
||||
break # Get only the first valid match per item
|
||||
# All matches should share the same mode
|
||||
if mode is None:
|
||||
mode = update.mode
|
||||
elif mode != update.mode:
|
||||
continue
|
||||
|
||||
mm_matches[(modality, item_idx)] = match, update_idx
|
||||
|
||||
# Prioritize earlier matches
|
||||
matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])
|
||||
@@ -885,47 +926,59 @@ def _iter_placeholders(
|
||||
prompt_len = len(prompt)
|
||||
start_idx = 0
|
||||
|
||||
while start_idx < prompt_len:
|
||||
found = False
|
||||
# The current (unfound) item's updates for each modality, with their
|
||||
# content resolved to token ids; rebuilt whenever an item is found.
|
||||
# Items are only resolved once the scan reaches them.
|
||||
candidates: list[tuple[str, ResolvedPromptUpdate, list[int]]] | None = None
|
||||
|
||||
for modality, modality_updates in mm_prompt_updates.items():
|
||||
item_idx = item_idx_by_modality[modality]
|
||||
if item_idx >= mm_item_counts.get(modality, 0):
|
||||
while start_idx < prompt_len:
|
||||
if candidates is None:
|
||||
candidates = [
|
||||
(modality, update, _seq2tokens(tokenizer, update.content.full))
|
||||
for modality, modality_updates in mm_prompt_updates.items()
|
||||
if item_idx_by_modality[modality] < mm_item_counts.get(modality, 0)
|
||||
for update in modality_updates[item_idx_by_modality[modality]]
|
||||
]
|
||||
if not candidates:
|
||||
return
|
||||
|
||||
found = False
|
||||
first_token = prompt[start_idx]
|
||||
|
||||
for modality, update, content_tokens_full in candidates:
|
||||
content_len_full = len(content_tokens_full)
|
||||
end_idx_full = start_idx + content_len_full
|
||||
|
||||
if content_len_full == 0 or end_idx_full > prompt_len:
|
||||
continue
|
||||
|
||||
for update in modality_updates[item_idx]:
|
||||
# Check the first token before comparing the full slice
|
||||
if (
|
||||
first_token == content_tokens_full[0]
|
||||
and prompt[start_idx:end_idx_full] == content_tokens_full
|
||||
):
|
||||
content = update.content
|
||||
content_tokens_full = _seq2tokens(tokenizer, content.full)
|
||||
content_len_full = len(content_tokens_full)
|
||||
end_idx_full = start_idx + content_len_full
|
||||
content_is_embed = content.is_embed
|
||||
if content_is_embed is not None:
|
||||
content_is_embed = content_is_embed(tokenizer, content.full)
|
||||
|
||||
if content_len_full == 0 or end_idx_full > prompt_len:
|
||||
continue
|
||||
yield PlaceholderFeaturesInfo(
|
||||
modality=modality,
|
||||
item_idx=item_idx_by_modality[modality],
|
||||
start_idx=start_idx,
|
||||
tokens=content_tokens_full,
|
||||
is_embed=content_is_embed,
|
||||
)
|
||||
|
||||
if prompt[start_idx:end_idx_full] == content_tokens_full:
|
||||
content_is_embed = content.is_embed
|
||||
if content_is_embed is not None:
|
||||
content_is_embed = content_is_embed(tokenizer, content.full)
|
||||
|
||||
yield PlaceholderFeaturesInfo(
|
||||
modality=modality,
|
||||
item_idx=item_idx,
|
||||
start_idx=start_idx,
|
||||
tokens=content_tokens_full,
|
||||
is_embed=content_is_embed,
|
||||
)
|
||||
|
||||
# Exclude overlapping matches
|
||||
start_idx = end_idx_full
|
||||
item_idx_by_modality[modality] += 1
|
||||
found = True
|
||||
break
|
||||
|
||||
if found:
|
||||
# Exclude overlapping matches
|
||||
start_idx = end_idx_full
|
||||
item_idx_by_modality[modality] += 1
|
||||
if _all_items_found(mm_item_counts, item_idx_by_modality):
|
||||
return
|
||||
|
||||
break # Go back to the outer while loop
|
||||
candidates = None
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
start_idx += 1
|
||||
|
||||
Reference in New Issue
Block a user