[Core][Multimodal] Skip redundant placeholder scan when token match succeeds (#52925)

Co-authored-by: shenyiqin <shenyiqin1@huawei.com>
This commit is contained in:
Yiqin
2026-08-20 15:04:29 +00:00
committed by GitHub
co-authored by shenyiqin
parent bd8865a299
commit cb09dd7488
4 changed files with 560 additions and 196 deletions
+369 -163
View File
@@ -17,6 +17,7 @@ from vllm.multimodal.processing.processor import (
PromptInsertion, PromptInsertion,
PromptReplacement, PromptReplacement,
_apply_matches, _apply_matches,
_apply_token_matches_with_placeholders,
apply_text_matches, apply_text_matches,
apply_token_matches, apply_token_matches,
find_mm_placeholders, find_mm_placeholders,
@@ -567,171 +568,309 @@ def test_find_update_text(
assert new_prompt == expected assert new_prompt == expected
FIND_UPDATE_TOKENS_TEST_CASES = [
# Tokenized test cases of `test_find_update_text`
# using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
(
[1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
{
# We use `<image>` before `Image:` to test matches that
# occur out of order
"pattern_1": [32000],
"pattern_2": [9833, 28747],
"pattern_3": [918],
},
{
# Test whether target is confused with replacement
"pattern_1": [32000, 32000],
# Test empty replacement
"pattern_2": [],
# Test dynamic replacement (beyond the form of `unit * count`)
"pattern_3": [1550, 918, 1550],
},
{
PromptInsertion: {
0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
1: [
1,
9833,
28747,
32000,
32000,
32000,
9833,
28747,
32000,
32000,
918,
1550,
918,
1550,
], # noqa: E501
2: [
1,
9833,
28747,
32000,
32000,
32000,
32000,
32000,
9833,
28747,
32000,
32000,
918,
1550,
918,
1550,
1550,
918,
1550,
], # noqa: E501
},
PromptReplacement: {
0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
1: [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550], # noqa: E501
2: [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
},
},
),
# Test index targets
(
[],
{
"pattern_1": PromptIndexTargets.start(),
"pattern_2": PromptIndexTargets.prefix([32000]),
"pattern_3": PromptIndexTargets.end(),
},
{
"pattern_1": [-1],
"pattern_2": [-2],
"pattern_3": [-3],
},
{
PromptInsertion: {
0: [],
1: [-1, -3],
2: [-1, -1, -3, -3],
},
PromptReplacement: {
0: [],
1: [-1, -3],
2: [-1, -1, -3, -3],
},
},
),
(
[32000],
{
"pattern_1": PromptIndexTargets.start(),
"pattern_2": PromptIndexTargets.prefix([32000]),
"pattern_3": PromptIndexTargets.end(),
},
{
"pattern_1": [-1],
"pattern_2": [-2],
"pattern_3": [-3],
},
{
PromptInsertion: {
0: [32000],
1: [-1, 32000, -2, -3],
2: [-1, -1, 32000, -2, -2, -3, -3],
},
PromptReplacement: {
0: [32000],
1: [-1, 32000, -2, -3],
2: [-1, -1, 32000, -2, -2, -3, -3],
},
},
),
# Test different replacement per item
(
[32000, 32000, 32000],
{
"pattern_1": [32000],
},
{
"pattern_1": lambda idx: [-(idx + 1)],
},
{
PromptInsertion: {
0: [32000, 32000, 32000],
1: [32000, -1, 32000, 32000],
2: [32000, -1, -2, 32000, 32000],
},
PromptReplacement: {
0: [32000, 32000, 32000],
1: [-1, 32000, 32000],
2: [-1, -2, 32000],
},
},
),
(
[32000, 32000, 32000],
{
"pattern_1": PromptIndexTargets.prefix([32000]),
},
{
"pattern_1": lambda idx: [-(idx + 1)],
},
{
PromptInsertion: {
0: [32000, 32000, 32000],
1: [32000, -1, 32000, 32000],
2: [32000, -1, -2, 32000, 32000],
},
PromptReplacement: {
0: [32000, 32000, 32000],
1: [32000, -1, 32000, 32000],
2: [32000, -1, -2, 32000, 32000],
},
},
),
]
def _placeholder(modality, item_idx, start_idx, tokens):
return PlaceholderFeaturesInfo(
modality=modality,
item_idx=item_idx,
start_idx=start_idx,
tokens=tokens,
is_embed=None,
)
FIND_UPDATE_TOKENS_PLACEHOLDER_EXPECTED = [
{
PromptInsertion: {
0: {},
1: {
"pattern_1": [_placeholder("pattern_1", 0, 4, [32000, 32000])],
"pattern_3": [_placeholder("pattern_3", 0, 11, [1550, 918, 1550])],
},
2: {
"pattern_1": [
_placeholder("pattern_1", 0, 4, [32000, 32000]),
_placeholder("pattern_1", 1, 6, [32000, 32000]),
],
"pattern_3": [
_placeholder("pattern_3", 0, 13, [1550, 918, 1550]),
_placeholder("pattern_3", 1, 16, [1550, 918, 1550]),
],
},
},
PromptReplacement: {
0: {},
1: {
"pattern_1": [_placeholder("pattern_1", 0, 1, [32000, 32000])],
"pattern_3": [_placeholder("pattern_3", 0, 7, [1550, 918, 1550])],
},
2: {},
},
},
{
PromptInsertion: {0: {}, 1: {}, 2: {}},
PromptReplacement: {0: {}, 1: {}, 2: {}},
},
{
PromptInsertion: {
0: {},
1: {
"pattern_1": [_placeholder("pattern_1", 0, 0, [-1])],
"pattern_2": [_placeholder("pattern_2", 0, 2, [-2])],
"pattern_3": [_placeholder("pattern_3", 0, 3, [-3])],
},
2: {
"pattern_1": [
_placeholder("pattern_1", 0, 0, [-1]),
_placeholder("pattern_1", 1, 1, [-1]),
],
"pattern_2": [
_placeholder("pattern_2", 0, 3, [-2]),
_placeholder("pattern_2", 1, 4, [-2]),
],
"pattern_3": [
_placeholder("pattern_3", 0, 5, [-3]),
_placeholder("pattern_3", 1, 6, [-3]),
],
},
},
PromptReplacement: {
0: {},
1: {
"pattern_1": [_placeholder("pattern_1", 0, 0, [-1])],
"pattern_2": [_placeholder("pattern_2", 0, 2, [-2])],
"pattern_3": [_placeholder("pattern_3", 0, 3, [-3])],
},
2: {
"pattern_1": [
_placeholder("pattern_1", 0, 0, [-1]),
_placeholder("pattern_1", 1, 1, [-1]),
],
"pattern_2": [
_placeholder("pattern_2", 0, 3, [-2]),
_placeholder("pattern_2", 1, 4, [-2]),
],
"pattern_3": [
_placeholder("pattern_3", 0, 5, [-3]),
_placeholder("pattern_3", 1, 6, [-3]),
],
},
},
},
{
PromptInsertion: {
0: {},
1: {"pattern_1": [_placeholder("pattern_1", 0, 1, [-1])]},
2: {
"pattern_1": [
_placeholder("pattern_1", 0, 1, [-1]),
_placeholder("pattern_1", 1, 2, [-2]),
]
},
},
PromptReplacement: {
0: {},
1: {"pattern_1": [_placeholder("pattern_1", 0, 0, [-1])]},
2: {
"pattern_1": [
_placeholder("pattern_1", 0, 0, [-1]),
_placeholder("pattern_1", 1, 1, [-2]),
]
},
},
},
{
PromptInsertion: {
0: {},
1: {"pattern_1": [_placeholder("pattern_1", 0, 1, [-1])]},
2: {
"pattern_1": [
_placeholder("pattern_1", 0, 1, [-1]),
_placeholder("pattern_1", 1, 2, [-2]),
]
},
},
PromptReplacement: {
0: {},
1: {"pattern_1": [_placeholder("pattern_1", 0, 1, [-1])]},
2: {
"pattern_1": [
_placeholder("pattern_1", 0, 1, [-1]),
_placeholder("pattern_1", 1, 2, [-2]),
]
},
},
},
]
@pytest.mark.parametrize( @pytest.mark.parametrize(
("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501 ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
[ FIND_UPDATE_TOKENS_TEST_CASES,
# Tokenized test cases of `test_find_update_text`
# using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
(
[1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
{
# We use `<image>` before `Image:` to test matches that
# occur out of order
"pattern_1": [32000],
"pattern_2": [9833, 28747],
"pattern_3": [918],
},
{
# Test whether target is confused with replacement
"pattern_1": [32000, 32000],
# Test empty replacement
"pattern_2": [],
# Test dynamic replacement (beyond the form of `unit * count`)
"pattern_3": [1550, 918, 1550],
},
{
PromptInsertion: {
0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
1: [
1,
9833,
28747,
32000,
32000,
32000,
9833,
28747,
32000,
32000,
918,
1550,
918,
1550,
], # noqa: E501
2: [
1,
9833,
28747,
32000,
32000,
32000,
32000,
32000,
9833,
28747,
32000,
32000,
918,
1550,
918,
1550,
1550,
918,
1550,
], # noqa: E501
},
PromptReplacement: {
0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
1: [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550], # noqa: E501
2: [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
},
},
),
# Test index targets
(
[],
{
"pattern_1": PromptIndexTargets.start(),
"pattern_2": PromptIndexTargets.prefix([32000]),
"pattern_3": PromptIndexTargets.end(),
},
{
"pattern_1": [-1],
"pattern_2": [-2],
"pattern_3": [-3],
},
{
PromptInsertion: {
0: [],
1: [-1, -3],
2: [-1, -1, -3, -3],
},
PromptReplacement: {
0: [],
1: [-1, -3],
2: [-1, -1, -3, -3],
},
},
),
(
[32000],
{
"pattern_1": PromptIndexTargets.start(),
"pattern_2": PromptIndexTargets.prefix([32000]),
"pattern_3": PromptIndexTargets.end(),
},
{
"pattern_1": [-1],
"pattern_2": [-2],
"pattern_3": [-3],
},
{
PromptInsertion: {
0: [32000],
1: [-1, 32000, -2, -3],
2: [-1, -1, 32000, -2, -2, -3, -3],
},
PromptReplacement: {
0: [32000],
1: [-1, 32000, -2, -3],
2: [-1, -1, 32000, -2, -2, -3, -3],
},
},
),
# Test different replacement per item
(
[32000, 32000, 32000],
{
"pattern_1": [32000],
},
{
"pattern_1": lambda idx: [-(idx + 1)],
},
{
PromptInsertion: {
0: [32000, 32000, 32000],
1: [32000, -1, 32000, 32000],
2: [32000, -1, -2, 32000, 32000],
},
PromptReplacement: {
0: [32000, 32000, 32000],
1: [-1, 32000, 32000],
2: [-1, -2, 32000],
},
},
),
(
[32000, 32000, 32000],
{
"pattern_1": PromptIndexTargets.prefix([32000]),
},
{
"pattern_1": lambda idx: [-(idx + 1)],
},
{
PromptInsertion: {
0: [32000, 32000, 32000],
1: [32000, -1, 32000, 32000],
2: [32000, -1, -2, 32000, 32000],
},
PromptReplacement: {
0: [32000, 32000, 32000],
1: [32000, -1, 32000, 32000],
2: [32000, -1, -2, 32000, 32000],
},
},
),
],
) )
def test_find_update_tokens( def test_find_update_tokens(
prompt, prompt,
@@ -769,6 +908,73 @@ def test_find_update_tokens(
assert new_prompt == expected assert new_prompt == expected
@pytest.mark.parametrize(
(
"prompt",
"target_by_key",
"repl_by_key",
"expected_by_update_type_mm_count",
"expected_placeholders_by_update_type_mm_count",
),
[
(*case, placeholder_expected)
for case, placeholder_expected in zip(
FIND_UPDATE_TOKENS_TEST_CASES,
FIND_UPDATE_TOKENS_PLACEHOLDER_EXPECTED,
strict=True,
)
],
)
def test_apply_token_matches_with_placeholders(
prompt,
target_by_key,
repl_by_key,
expected_by_update_type_mm_count,
expected_placeholders_by_update_type_mm_count,
):
for update_type, expected_by_mm_count in expected_by_update_type_mm_count.items():
for mm_count, expected in expected_by_mm_count.items():
mm_prompt_updates = {
key: [
[update_type(key, target, repl_by_key[key]).resolve(i)]
for i in range(mm_count)
]
for key, target in target_by_key.items()
}
new_prompt, result, placeholders = _apply_token_matches_with_placeholders(
prompt,
mm_prompt_updates,
tokenizer=None,
)
if any(
update_idx is None
for update_idxs in result.values()
for update_idx in update_idxs
):
continue
expected_placeholders = expected_placeholders_by_update_type_mm_count[
update_type
][mm_count]
# Only displayed on error
print("update_type:", update_type)
print("mm_count:", mm_count)
print("mm_prompt_updates:", mm_prompt_updates)
print("new_prompt:", new_prompt)
print("result:", result)
print("placeholders:", placeholders)
assert new_prompt == expected
assert {
modality: ph_list
for modality, ph_list in placeholders.items()
if ph_list
} == expected_placeholders
@pytest.mark.parametrize( @pytest.mark.parametrize(
"repl_by_key", "repl_by_key",
[ [
+34
View File
@@ -373,6 +373,40 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
return token_ids, res return token_ids, res
def _apply_token_matches_with_placeholders(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[
list[int],
MultiModalPromptUpdatesApplyResult,
Mapping[str, list[PlaceholderFeaturesInfo]],
]:
new_token_ids, match_result = self._apply_token_matches(
token_ids,
mm_prompt_updates,
)
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
modality: [] for modality in mm_prompt_updates
}
if all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
placeholders = dict(
self._find_mm_placeholders(
new_token_ids,
self._matched_updates_from_result(
mm_prompt_updates,
match_result,
),
)
)
return new_token_ids, match_result, placeholders
def _find_mm_placeholders( def _find_mm_placeholders(
self, self,
new_token_ids: list[int], new_token_ids: list[int],
+34
View File
@@ -391,6 +391,40 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
return token_ids, res return token_ids, res
def _apply_token_matches_with_placeholders(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[
list[int],
MultiModalPromptUpdatesApplyResult,
Mapping[str, list[PlaceholderFeaturesInfo]],
]:
new_token_ids, match_result = self._apply_token_matches(
token_ids,
mm_prompt_updates,
)
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
modality: [] for modality in mm_prompt_updates
}
if all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
placeholders = dict(
self._find_mm_placeholders(
new_token_ids,
self._matched_updates_from_result(
mm_prompt_updates,
match_result,
),
)
)
return new_token_ids, match_result, placeholders
def _find_mm_placeholders( def _find_mm_placeholders(
self, self,
new_token_ids: list[int], new_token_ids: list[int],
+123 -33
View File
@@ -958,6 +958,64 @@ def apply_token_matches(
return flatten_2d_lists(token_id_seqs), result return flatten_2d_lists(token_id_seqs), result
def _apply_token_matches_with_placeholders(
token_ids: list[int],
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: TokenizerLike | None,
) -> tuple[
list[int],
"MultiModalPromptUpdatesApplyResult",
Mapping[str, list[PlaceholderFeaturesInfo]],
]:
matched_updates, result = _plan_prompt_updates(
token_ids,
mm_prompt_updates,
tokenizer,
)
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
modality: [] for modality in mm_prompt_updates
}
new_token_ids = list[int]()
prev_end_idx = 0
for matched_update in matched_updates:
update = matched_update.update
match = matched_update.match
matched_content = update.content.full
if update.mode == UpdateMode.INSERT:
end_idx_to_insert = match.end_idx
elif update.mode == UpdateMode.REPLACE:
end_idx_to_insert = match.start_idx
else:
assert_never(update.mode)
new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
start_idx = len(new_token_ids)
tokens = _seq2tokens(tokenizer, matched_content)
if tokens:
content_is_embed = update.content.is_embed
if content_is_embed is not None:
content_is_embed = content_is_embed(tokenizer, matched_content)
placeholders[update.modality].append(
PlaceholderFeaturesInfo(
modality=update.modality,
item_idx=update.item_idx,
start_idx=start_idx,
tokens=tokens,
is_embed=content_is_embed,
)
)
new_token_ids.extend(tokens)
prev_end_idx = match.end_idx
new_token_ids.extend(token_ids[prev_end_idx:])
return new_token_ids, result, placeholders
def apply_text_matches( def apply_text_matches(
prompt: str, prompt: str,
mm_prompt_updates: "MultiModalPromptUpdates", mm_prompt_updates: "MultiModalPromptUpdates",
@@ -1671,6 +1729,22 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
return apply_token_matches(prompt, mm_prompt_updates, tokenizer) return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
def _apply_token_matches_with_placeholders(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[
list[int],
MultiModalPromptUpdatesApplyResult,
Mapping[str, list[PlaceholderFeaturesInfo]],
]:
tokenizer = self.info.get_tokenizer()
return _apply_token_matches_with_placeholders(
token_ids,
mm_prompt_updates,
tokenizer,
)
def _apply_text_matches( def _apply_text_matches(
self, self,
prompt: str, prompt: str,
@@ -1689,40 +1763,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt, mm_prompt_updates, tokenizer prompt, mm_prompt_updates, tokenizer
) )
def _apply_prompt_updates( def _matched_updates_from_result(
self, self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates, mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: match_result: MultiModalPromptUpdatesApplyResult,
"""Apply multi-modal prompt updates to token IDs.""" ) -> dict[str, list[Sequence[ResolvedPromptUpdate]]]:
tokenizer = self.info.get_tokenizer()
new_token_ids, match_result = self._apply_token_matches(
token_ids,
mm_prompt_updates,
)
# If the search text does not represent a special token,
# it may have different token IDs in the prompt, because
# the tokens may go across the boundaries of the search text.
# ----
# e.g. when searching for "foo" in "food", if "food" itself makes
# up a token, then the token ID of "foo" will not appear at all
# ----
# Since it is inefficient to search for all possible tokenizations
# of the search text in the prompt, we instead perform string-based
# updates on the decoded token IDs, then encode them back.
if not all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
new_text, match_result = self._apply_text_matches(
_seq2text(tokenizer, token_ids, use_cache=False),
mm_prompt_updates,
)
new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list) matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
for modality, update_idxs in match_result.items(): for modality, update_idxs in match_result.items():
for item_idx, update_idx in enumerate(update_idxs): for item_idx, update_idx in enumerate(update_idxs):
@@ -1735,9 +1780,54 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
[mm_prompt_updates[modality][item_idx][update_idx]] [mm_prompt_updates[modality][item_idx][update_idx]]
) )
return dict(matched_updates)
def _apply_prompt_updates(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""Apply multi-modal prompt updates to token IDs."""
tokenizer = self.info.get_tokenizer()
new_token_ids, match_result, placeholders = (
self._apply_token_matches_with_placeholders(
token_ids,
mm_prompt_updates,
)
)
if all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
placeholders = {
modality: modality_placeholders
for modality, modality_placeholders in placeholders.items()
if modality_placeholders
}
return new_token_ids, placeholders
# If the search text does not represent a special token,
# it may have different token IDs in the prompt, because
# the tokens may go across the boundaries of the search text.
# ----
# e.g. when searching for "foo" in "food", if "food" itself makes
# up a token, then the token ID of "foo" will not appear at all
# ----
# Since it is inefficient to search for all possible tokenizations
# of the search text in the prompt, we instead perform string-based
# updates on the decoded token IDs, then encode them back.
new_text, match_result = self._apply_text_matches(
_seq2text(tokenizer, token_ids, use_cache=False),
mm_prompt_updates,
)
new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
placeholders = self._find_mm_placeholders( placeholders = self._find_mm_placeholders(
new_token_ids, new_token_ids,
dict(matched_updates), self._matched_updates_from_result(mm_prompt_updates, match_result),
) )
return new_token_ids, placeholders return new_token_ids, placeholders