[Core][Multimodal] Skip redundant placeholder scan when token match succeeds (#52925)
Co-authored-by: shenyiqin <shenyiqin1@huawei.com>
This commit is contained in:
+369
-163
@@ -17,6 +17,7 @@ from vllm.multimodal.processing.processor import (
|
|||||||
PromptInsertion,
|
PromptInsertion,
|
||||||
PromptReplacement,
|
PromptReplacement,
|
||||||
_apply_matches,
|
_apply_matches,
|
||||||
|
_apply_token_matches_with_placeholders,
|
||||||
apply_text_matches,
|
apply_text_matches,
|
||||||
apply_token_matches,
|
apply_token_matches,
|
||||||
find_mm_placeholders,
|
find_mm_placeholders,
|
||||||
@@ -567,171 +568,309 @@ def test_find_update_text(
|
|||||||
assert new_prompt == expected
|
assert new_prompt == expected
|
||||||
|
|
||||||
|
|
||||||
|
FIND_UPDATE_TOKENS_TEST_CASES = [
|
||||||
|
# Tokenized test cases of `test_find_update_text`
|
||||||
|
# using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
|
||||||
|
(
|
||||||
|
[1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
|
||||||
|
{
|
||||||
|
# We use `<image>` before `Image:` to test matches that
|
||||||
|
# occur out of order
|
||||||
|
"pattern_1": [32000],
|
||||||
|
"pattern_2": [9833, 28747],
|
||||||
|
"pattern_3": [918],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
# Test whether target is confused with replacement
|
||||||
|
"pattern_1": [32000, 32000],
|
||||||
|
# Test empty replacement
|
||||||
|
"pattern_2": [],
|
||||||
|
# Test dynamic replacement (beyond the form of `unit * count`)
|
||||||
|
"pattern_3": [1550, 918, 1550],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
PromptInsertion: {
|
||||||
|
0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
|
||||||
|
1: [
|
||||||
|
1,
|
||||||
|
9833,
|
||||||
|
28747,
|
||||||
|
32000,
|
||||||
|
32000,
|
||||||
|
32000,
|
||||||
|
9833,
|
||||||
|
28747,
|
||||||
|
32000,
|
||||||
|
32000,
|
||||||
|
918,
|
||||||
|
1550,
|
||||||
|
918,
|
||||||
|
1550,
|
||||||
|
], # noqa: E501
|
||||||
|
2: [
|
||||||
|
1,
|
||||||
|
9833,
|
||||||
|
28747,
|
||||||
|
32000,
|
||||||
|
32000,
|
||||||
|
32000,
|
||||||
|
32000,
|
||||||
|
32000,
|
||||||
|
9833,
|
||||||
|
28747,
|
||||||
|
32000,
|
||||||
|
32000,
|
||||||
|
918,
|
||||||
|
1550,
|
||||||
|
918,
|
||||||
|
1550,
|
||||||
|
1550,
|
||||||
|
918,
|
||||||
|
1550,
|
||||||
|
], # noqa: E501
|
||||||
|
},
|
||||||
|
PromptReplacement: {
|
||||||
|
0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
|
||||||
|
1: [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550], # noqa: E501
|
||||||
|
2: [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
# Test index targets
|
||||||
|
(
|
||||||
|
[],
|
||||||
|
{
|
||||||
|
"pattern_1": PromptIndexTargets.start(),
|
||||||
|
"pattern_2": PromptIndexTargets.prefix([32000]),
|
||||||
|
"pattern_3": PromptIndexTargets.end(),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"pattern_1": [-1],
|
||||||
|
"pattern_2": [-2],
|
||||||
|
"pattern_3": [-3],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
PromptInsertion: {
|
||||||
|
0: [],
|
||||||
|
1: [-1, -3],
|
||||||
|
2: [-1, -1, -3, -3],
|
||||||
|
},
|
||||||
|
PromptReplacement: {
|
||||||
|
0: [],
|
||||||
|
1: [-1, -3],
|
||||||
|
2: [-1, -1, -3, -3],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
(
|
||||||
|
[32000],
|
||||||
|
{
|
||||||
|
"pattern_1": PromptIndexTargets.start(),
|
||||||
|
"pattern_2": PromptIndexTargets.prefix([32000]),
|
||||||
|
"pattern_3": PromptIndexTargets.end(),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"pattern_1": [-1],
|
||||||
|
"pattern_2": [-2],
|
||||||
|
"pattern_3": [-3],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
PromptInsertion: {
|
||||||
|
0: [32000],
|
||||||
|
1: [-1, 32000, -2, -3],
|
||||||
|
2: [-1, -1, 32000, -2, -2, -3, -3],
|
||||||
|
},
|
||||||
|
PromptReplacement: {
|
||||||
|
0: [32000],
|
||||||
|
1: [-1, 32000, -2, -3],
|
||||||
|
2: [-1, -1, 32000, -2, -2, -3, -3],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
# Test different replacement per item
|
||||||
|
(
|
||||||
|
[32000, 32000, 32000],
|
||||||
|
{
|
||||||
|
"pattern_1": [32000],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"pattern_1": lambda idx: [-(idx + 1)],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
PromptInsertion: {
|
||||||
|
0: [32000, 32000, 32000],
|
||||||
|
1: [32000, -1, 32000, 32000],
|
||||||
|
2: [32000, -1, -2, 32000, 32000],
|
||||||
|
},
|
||||||
|
PromptReplacement: {
|
||||||
|
0: [32000, 32000, 32000],
|
||||||
|
1: [-1, 32000, 32000],
|
||||||
|
2: [-1, -2, 32000],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
(
|
||||||
|
[32000, 32000, 32000],
|
||||||
|
{
|
||||||
|
"pattern_1": PromptIndexTargets.prefix([32000]),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"pattern_1": lambda idx: [-(idx + 1)],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
PromptInsertion: {
|
||||||
|
0: [32000, 32000, 32000],
|
||||||
|
1: [32000, -1, 32000, 32000],
|
||||||
|
2: [32000, -1, -2, 32000, 32000],
|
||||||
|
},
|
||||||
|
PromptReplacement: {
|
||||||
|
0: [32000, 32000, 32000],
|
||||||
|
1: [32000, -1, 32000, 32000],
|
||||||
|
2: [32000, -1, -2, 32000, 32000],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _placeholder(modality, item_idx, start_idx, tokens):
|
||||||
|
return PlaceholderFeaturesInfo(
|
||||||
|
modality=modality,
|
||||||
|
item_idx=item_idx,
|
||||||
|
start_idx=start_idx,
|
||||||
|
tokens=tokens,
|
||||||
|
is_embed=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
FIND_UPDATE_TOKENS_PLACEHOLDER_EXPECTED = [
|
||||||
|
{
|
||||||
|
PromptInsertion: {
|
||||||
|
0: {},
|
||||||
|
1: {
|
||||||
|
"pattern_1": [_placeholder("pattern_1", 0, 4, [32000, 32000])],
|
||||||
|
"pattern_3": [_placeholder("pattern_3", 0, 11, [1550, 918, 1550])],
|
||||||
|
},
|
||||||
|
2: {
|
||||||
|
"pattern_1": [
|
||||||
|
_placeholder("pattern_1", 0, 4, [32000, 32000]),
|
||||||
|
_placeholder("pattern_1", 1, 6, [32000, 32000]),
|
||||||
|
],
|
||||||
|
"pattern_3": [
|
||||||
|
_placeholder("pattern_3", 0, 13, [1550, 918, 1550]),
|
||||||
|
_placeholder("pattern_3", 1, 16, [1550, 918, 1550]),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
PromptReplacement: {
|
||||||
|
0: {},
|
||||||
|
1: {
|
||||||
|
"pattern_1": [_placeholder("pattern_1", 0, 1, [32000, 32000])],
|
||||||
|
"pattern_3": [_placeholder("pattern_3", 0, 7, [1550, 918, 1550])],
|
||||||
|
},
|
||||||
|
2: {},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
PromptInsertion: {0: {}, 1: {}, 2: {}},
|
||||||
|
PromptReplacement: {0: {}, 1: {}, 2: {}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
PromptInsertion: {
|
||||||
|
0: {},
|
||||||
|
1: {
|
||||||
|
"pattern_1": [_placeholder("pattern_1", 0, 0, [-1])],
|
||||||
|
"pattern_2": [_placeholder("pattern_2", 0, 2, [-2])],
|
||||||
|
"pattern_3": [_placeholder("pattern_3", 0, 3, [-3])],
|
||||||
|
},
|
||||||
|
2: {
|
||||||
|
"pattern_1": [
|
||||||
|
_placeholder("pattern_1", 0, 0, [-1]),
|
||||||
|
_placeholder("pattern_1", 1, 1, [-1]),
|
||||||
|
],
|
||||||
|
"pattern_2": [
|
||||||
|
_placeholder("pattern_2", 0, 3, [-2]),
|
||||||
|
_placeholder("pattern_2", 1, 4, [-2]),
|
||||||
|
],
|
||||||
|
"pattern_3": [
|
||||||
|
_placeholder("pattern_3", 0, 5, [-3]),
|
||||||
|
_placeholder("pattern_3", 1, 6, [-3]),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
PromptReplacement: {
|
||||||
|
0: {},
|
||||||
|
1: {
|
||||||
|
"pattern_1": [_placeholder("pattern_1", 0, 0, [-1])],
|
||||||
|
"pattern_2": [_placeholder("pattern_2", 0, 2, [-2])],
|
||||||
|
"pattern_3": [_placeholder("pattern_3", 0, 3, [-3])],
|
||||||
|
},
|
||||||
|
2: {
|
||||||
|
"pattern_1": [
|
||||||
|
_placeholder("pattern_1", 0, 0, [-1]),
|
||||||
|
_placeholder("pattern_1", 1, 1, [-1]),
|
||||||
|
],
|
||||||
|
"pattern_2": [
|
||||||
|
_placeholder("pattern_2", 0, 3, [-2]),
|
||||||
|
_placeholder("pattern_2", 1, 4, [-2]),
|
||||||
|
],
|
||||||
|
"pattern_3": [
|
||||||
|
_placeholder("pattern_3", 0, 5, [-3]),
|
||||||
|
_placeholder("pattern_3", 1, 6, [-3]),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
PromptInsertion: {
|
||||||
|
0: {},
|
||||||
|
1: {"pattern_1": [_placeholder("pattern_1", 0, 1, [-1])]},
|
||||||
|
2: {
|
||||||
|
"pattern_1": [
|
||||||
|
_placeholder("pattern_1", 0, 1, [-1]),
|
||||||
|
_placeholder("pattern_1", 1, 2, [-2]),
|
||||||
|
]
|
||||||
|
},
|
||||||
|
},
|
||||||
|
PromptReplacement: {
|
||||||
|
0: {},
|
||||||
|
1: {"pattern_1": [_placeholder("pattern_1", 0, 0, [-1])]},
|
||||||
|
2: {
|
||||||
|
"pattern_1": [
|
||||||
|
_placeholder("pattern_1", 0, 0, [-1]),
|
||||||
|
_placeholder("pattern_1", 1, 1, [-2]),
|
||||||
|
]
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
PromptInsertion: {
|
||||||
|
0: {},
|
||||||
|
1: {"pattern_1": [_placeholder("pattern_1", 0, 1, [-1])]},
|
||||||
|
2: {
|
||||||
|
"pattern_1": [
|
||||||
|
_placeholder("pattern_1", 0, 1, [-1]),
|
||||||
|
_placeholder("pattern_1", 1, 2, [-2]),
|
||||||
|
]
|
||||||
|
},
|
||||||
|
},
|
||||||
|
PromptReplacement: {
|
||||||
|
0: {},
|
||||||
|
1: {"pattern_1": [_placeholder("pattern_1", 0, 1, [-1])]},
|
||||||
|
2: {
|
||||||
|
"pattern_1": [
|
||||||
|
_placeholder("pattern_1", 0, 1, [-1]),
|
||||||
|
_placeholder("pattern_1", 1, 2, [-2]),
|
||||||
|
]
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
|
("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
|
||||||
[
|
FIND_UPDATE_TOKENS_TEST_CASES,
|
||||||
# Tokenized test cases of `test_find_update_text`
|
|
||||||
# using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
|
|
||||||
(
|
|
||||||
[1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
|
|
||||||
{
|
|
||||||
# We use `<image>` before `Image:` to test matches that
|
|
||||||
# occur out of order
|
|
||||||
"pattern_1": [32000],
|
|
||||||
"pattern_2": [9833, 28747],
|
|
||||||
"pattern_3": [918],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
# Test whether target is confused with replacement
|
|
||||||
"pattern_1": [32000, 32000],
|
|
||||||
# Test empty replacement
|
|
||||||
"pattern_2": [],
|
|
||||||
# Test dynamic replacement (beyond the form of `unit * count`)
|
|
||||||
"pattern_3": [1550, 918, 1550],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
PromptInsertion: {
|
|
||||||
0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
|
|
||||||
1: [
|
|
||||||
1,
|
|
||||||
9833,
|
|
||||||
28747,
|
|
||||||
32000,
|
|
||||||
32000,
|
|
||||||
32000,
|
|
||||||
9833,
|
|
||||||
28747,
|
|
||||||
32000,
|
|
||||||
32000,
|
|
||||||
918,
|
|
||||||
1550,
|
|
||||||
918,
|
|
||||||
1550,
|
|
||||||
], # noqa: E501
|
|
||||||
2: [
|
|
||||||
1,
|
|
||||||
9833,
|
|
||||||
28747,
|
|
||||||
32000,
|
|
||||||
32000,
|
|
||||||
32000,
|
|
||||||
32000,
|
|
||||||
32000,
|
|
||||||
9833,
|
|
||||||
28747,
|
|
||||||
32000,
|
|
||||||
32000,
|
|
||||||
918,
|
|
||||||
1550,
|
|
||||||
918,
|
|
||||||
1550,
|
|
||||||
1550,
|
|
||||||
918,
|
|
||||||
1550,
|
|
||||||
], # noqa: E501
|
|
||||||
},
|
|
||||||
PromptReplacement: {
|
|
||||||
0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
|
|
||||||
1: [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550], # noqa: E501
|
|
||||||
2: [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
),
|
|
||||||
# Test index targets
|
|
||||||
(
|
|
||||||
[],
|
|
||||||
{
|
|
||||||
"pattern_1": PromptIndexTargets.start(),
|
|
||||||
"pattern_2": PromptIndexTargets.prefix([32000]),
|
|
||||||
"pattern_3": PromptIndexTargets.end(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"pattern_1": [-1],
|
|
||||||
"pattern_2": [-2],
|
|
||||||
"pattern_3": [-3],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
PromptInsertion: {
|
|
||||||
0: [],
|
|
||||||
1: [-1, -3],
|
|
||||||
2: [-1, -1, -3, -3],
|
|
||||||
},
|
|
||||||
PromptReplacement: {
|
|
||||||
0: [],
|
|
||||||
1: [-1, -3],
|
|
||||||
2: [-1, -1, -3, -3],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
),
|
|
||||||
(
|
|
||||||
[32000],
|
|
||||||
{
|
|
||||||
"pattern_1": PromptIndexTargets.start(),
|
|
||||||
"pattern_2": PromptIndexTargets.prefix([32000]),
|
|
||||||
"pattern_3": PromptIndexTargets.end(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"pattern_1": [-1],
|
|
||||||
"pattern_2": [-2],
|
|
||||||
"pattern_3": [-3],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
PromptInsertion: {
|
|
||||||
0: [32000],
|
|
||||||
1: [-1, 32000, -2, -3],
|
|
||||||
2: [-1, -1, 32000, -2, -2, -3, -3],
|
|
||||||
},
|
|
||||||
PromptReplacement: {
|
|
||||||
0: [32000],
|
|
||||||
1: [-1, 32000, -2, -3],
|
|
||||||
2: [-1, -1, 32000, -2, -2, -3, -3],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
),
|
|
||||||
# Test different replacement per item
|
|
||||||
(
|
|
||||||
[32000, 32000, 32000],
|
|
||||||
{
|
|
||||||
"pattern_1": [32000],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"pattern_1": lambda idx: [-(idx + 1)],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
PromptInsertion: {
|
|
||||||
0: [32000, 32000, 32000],
|
|
||||||
1: [32000, -1, 32000, 32000],
|
|
||||||
2: [32000, -1, -2, 32000, 32000],
|
|
||||||
},
|
|
||||||
PromptReplacement: {
|
|
||||||
0: [32000, 32000, 32000],
|
|
||||||
1: [-1, 32000, 32000],
|
|
||||||
2: [-1, -2, 32000],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
),
|
|
||||||
(
|
|
||||||
[32000, 32000, 32000],
|
|
||||||
{
|
|
||||||
"pattern_1": PromptIndexTargets.prefix([32000]),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"pattern_1": lambda idx: [-(idx + 1)],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
PromptInsertion: {
|
|
||||||
0: [32000, 32000, 32000],
|
|
||||||
1: [32000, -1, 32000, 32000],
|
|
||||||
2: [32000, -1, -2, 32000, 32000],
|
|
||||||
},
|
|
||||||
PromptReplacement: {
|
|
||||||
0: [32000, 32000, 32000],
|
|
||||||
1: [32000, -1, 32000, 32000],
|
|
||||||
2: [32000, -1, -2, 32000, 32000],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
),
|
|
||||||
],
|
|
||||||
)
|
)
|
||||||
def test_find_update_tokens(
|
def test_find_update_tokens(
|
||||||
prompt,
|
prompt,
|
||||||
@@ -769,6 +908,73 @@ def test_find_update_tokens(
|
|||||||
assert new_prompt == expected
|
assert new_prompt == expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
(
|
||||||
|
"prompt",
|
||||||
|
"target_by_key",
|
||||||
|
"repl_by_key",
|
||||||
|
"expected_by_update_type_mm_count",
|
||||||
|
"expected_placeholders_by_update_type_mm_count",
|
||||||
|
),
|
||||||
|
[
|
||||||
|
(*case, placeholder_expected)
|
||||||
|
for case, placeholder_expected in zip(
|
||||||
|
FIND_UPDATE_TOKENS_TEST_CASES,
|
||||||
|
FIND_UPDATE_TOKENS_PLACEHOLDER_EXPECTED,
|
||||||
|
strict=True,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_apply_token_matches_with_placeholders(
|
||||||
|
prompt,
|
||||||
|
target_by_key,
|
||||||
|
repl_by_key,
|
||||||
|
expected_by_update_type_mm_count,
|
||||||
|
expected_placeholders_by_update_type_mm_count,
|
||||||
|
):
|
||||||
|
for update_type, expected_by_mm_count in expected_by_update_type_mm_count.items():
|
||||||
|
for mm_count, expected in expected_by_mm_count.items():
|
||||||
|
mm_prompt_updates = {
|
||||||
|
key: [
|
||||||
|
[update_type(key, target, repl_by_key[key]).resolve(i)]
|
||||||
|
for i in range(mm_count)
|
||||||
|
]
|
||||||
|
for key, target in target_by_key.items()
|
||||||
|
}
|
||||||
|
|
||||||
|
new_prompt, result, placeholders = _apply_token_matches_with_placeholders(
|
||||||
|
prompt,
|
||||||
|
mm_prompt_updates,
|
||||||
|
tokenizer=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
if any(
|
||||||
|
update_idx is None
|
||||||
|
for update_idxs in result.values()
|
||||||
|
for update_idx in update_idxs
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
|
expected_placeholders = expected_placeholders_by_update_type_mm_count[
|
||||||
|
update_type
|
||||||
|
][mm_count]
|
||||||
|
|
||||||
|
# Only displayed on error
|
||||||
|
print("update_type:", update_type)
|
||||||
|
print("mm_count:", mm_count)
|
||||||
|
print("mm_prompt_updates:", mm_prompt_updates)
|
||||||
|
print("new_prompt:", new_prompt)
|
||||||
|
print("result:", result)
|
||||||
|
print("placeholders:", placeholders)
|
||||||
|
|
||||||
|
assert new_prompt == expected
|
||||||
|
assert {
|
||||||
|
modality: ph_list
|
||||||
|
for modality, ph_list in placeholders.items()
|
||||||
|
if ph_list
|
||||||
|
} == expected_placeholders
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"repl_by_key",
|
"repl_by_key",
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -373,6 +373,40 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
|||||||
|
|
||||||
return token_ids, res
|
return token_ids, res
|
||||||
|
|
||||||
|
def _apply_token_matches_with_placeholders(
|
||||||
|
self,
|
||||||
|
token_ids: list[int],
|
||||||
|
mm_prompt_updates: MultiModalPromptUpdates,
|
||||||
|
) -> tuple[
|
||||||
|
list[int],
|
||||||
|
MultiModalPromptUpdatesApplyResult,
|
||||||
|
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||||
|
]:
|
||||||
|
new_token_ids, match_result = self._apply_token_matches(
|
||||||
|
token_ids,
|
||||||
|
mm_prompt_updates,
|
||||||
|
)
|
||||||
|
|
||||||
|
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
|
||||||
|
modality: [] for modality in mm_prompt_updates
|
||||||
|
}
|
||||||
|
|
||||||
|
if all(
|
||||||
|
all(update_idx is not None for update_idx in update_idxs)
|
||||||
|
for update_idxs in match_result.values()
|
||||||
|
):
|
||||||
|
placeholders = dict(
|
||||||
|
self._find_mm_placeholders(
|
||||||
|
new_token_ids,
|
||||||
|
self._matched_updates_from_result(
|
||||||
|
mm_prompt_updates,
|
||||||
|
match_result,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return new_token_ids, match_result, placeholders
|
||||||
|
|
||||||
def _find_mm_placeholders(
|
def _find_mm_placeholders(
|
||||||
self,
|
self,
|
||||||
new_token_ids: list[int],
|
new_token_ids: list[int],
|
||||||
|
|||||||
@@ -391,6 +391,40 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
|||||||
|
|
||||||
return token_ids, res
|
return token_ids, res
|
||||||
|
|
||||||
|
def _apply_token_matches_with_placeholders(
|
||||||
|
self,
|
||||||
|
token_ids: list[int],
|
||||||
|
mm_prompt_updates: MultiModalPromptUpdates,
|
||||||
|
) -> tuple[
|
||||||
|
list[int],
|
||||||
|
MultiModalPromptUpdatesApplyResult,
|
||||||
|
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||||
|
]:
|
||||||
|
new_token_ids, match_result = self._apply_token_matches(
|
||||||
|
token_ids,
|
||||||
|
mm_prompt_updates,
|
||||||
|
)
|
||||||
|
|
||||||
|
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
|
||||||
|
modality: [] for modality in mm_prompt_updates
|
||||||
|
}
|
||||||
|
|
||||||
|
if all(
|
||||||
|
all(update_idx is not None for update_idx in update_idxs)
|
||||||
|
for update_idxs in match_result.values()
|
||||||
|
):
|
||||||
|
placeholders = dict(
|
||||||
|
self._find_mm_placeholders(
|
||||||
|
new_token_ids,
|
||||||
|
self._matched_updates_from_result(
|
||||||
|
mm_prompt_updates,
|
||||||
|
match_result,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return new_token_ids, match_result, placeholders
|
||||||
|
|
||||||
def _find_mm_placeholders(
|
def _find_mm_placeholders(
|
||||||
self,
|
self,
|
||||||
new_token_ids: list[int],
|
new_token_ids: list[int],
|
||||||
|
|||||||
@@ -958,6 +958,64 @@ def apply_token_matches(
|
|||||||
return flatten_2d_lists(token_id_seqs), result
|
return flatten_2d_lists(token_id_seqs), result
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_token_matches_with_placeholders(
|
||||||
|
token_ids: list[int],
|
||||||
|
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||||
|
tokenizer: TokenizerLike | None,
|
||||||
|
) -> tuple[
|
||||||
|
list[int],
|
||||||
|
"MultiModalPromptUpdatesApplyResult",
|
||||||
|
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||||
|
]:
|
||||||
|
matched_updates, result = _plan_prompt_updates(
|
||||||
|
token_ids,
|
||||||
|
mm_prompt_updates,
|
||||||
|
tokenizer,
|
||||||
|
)
|
||||||
|
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
|
||||||
|
modality: [] for modality in mm_prompt_updates
|
||||||
|
}
|
||||||
|
|
||||||
|
new_token_ids = list[int]()
|
||||||
|
prev_end_idx = 0
|
||||||
|
for matched_update in matched_updates:
|
||||||
|
update = matched_update.update
|
||||||
|
match = matched_update.match
|
||||||
|
matched_content = update.content.full
|
||||||
|
|
||||||
|
if update.mode == UpdateMode.INSERT:
|
||||||
|
end_idx_to_insert = match.end_idx
|
||||||
|
elif update.mode == UpdateMode.REPLACE:
|
||||||
|
end_idx_to_insert = match.start_idx
|
||||||
|
else:
|
||||||
|
assert_never(update.mode)
|
||||||
|
|
||||||
|
new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
|
||||||
|
start_idx = len(new_token_ids)
|
||||||
|
|
||||||
|
tokens = _seq2tokens(tokenizer, matched_content)
|
||||||
|
if tokens:
|
||||||
|
content_is_embed = update.content.is_embed
|
||||||
|
if content_is_embed is not None:
|
||||||
|
content_is_embed = content_is_embed(tokenizer, matched_content)
|
||||||
|
|
||||||
|
placeholders[update.modality].append(
|
||||||
|
PlaceholderFeaturesInfo(
|
||||||
|
modality=update.modality,
|
||||||
|
item_idx=update.item_idx,
|
||||||
|
start_idx=start_idx,
|
||||||
|
tokens=tokens,
|
||||||
|
is_embed=content_is_embed,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
new_token_ids.extend(tokens)
|
||||||
|
|
||||||
|
prev_end_idx = match.end_idx
|
||||||
|
|
||||||
|
new_token_ids.extend(token_ids[prev_end_idx:])
|
||||||
|
return new_token_ids, result, placeholders
|
||||||
|
|
||||||
|
|
||||||
def apply_text_matches(
|
def apply_text_matches(
|
||||||
prompt: str,
|
prompt: str,
|
||||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||||
@@ -1671,6 +1729,22 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
|
return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
|
||||||
|
|
||||||
|
def _apply_token_matches_with_placeholders(
|
||||||
|
self,
|
||||||
|
token_ids: list[int],
|
||||||
|
mm_prompt_updates: MultiModalPromptUpdates,
|
||||||
|
) -> tuple[
|
||||||
|
list[int],
|
||||||
|
MultiModalPromptUpdatesApplyResult,
|
||||||
|
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||||
|
]:
|
||||||
|
tokenizer = self.info.get_tokenizer()
|
||||||
|
return _apply_token_matches_with_placeholders(
|
||||||
|
token_ids,
|
||||||
|
mm_prompt_updates,
|
||||||
|
tokenizer,
|
||||||
|
)
|
||||||
|
|
||||||
def _apply_text_matches(
|
def _apply_text_matches(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
prompt: str,
|
||||||
@@ -1689,40 +1763,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
prompt, mm_prompt_updates, tokenizer
|
prompt, mm_prompt_updates, tokenizer
|
||||||
)
|
)
|
||||||
|
|
||||||
def _apply_prompt_updates(
|
def _matched_updates_from_result(
|
||||||
self,
|
self,
|
||||||
token_ids: list[int],
|
|
||||||
mm_prompt_updates: MultiModalPromptUpdates,
|
mm_prompt_updates: MultiModalPromptUpdates,
|
||||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
match_result: MultiModalPromptUpdatesApplyResult,
|
||||||
"""Apply multi-modal prompt updates to token IDs."""
|
) -> dict[str, list[Sequence[ResolvedPromptUpdate]]]:
|
||||||
tokenizer = self.info.get_tokenizer()
|
|
||||||
|
|
||||||
new_token_ids, match_result = self._apply_token_matches(
|
|
||||||
token_ids,
|
|
||||||
mm_prompt_updates,
|
|
||||||
)
|
|
||||||
|
|
||||||
# If the search text does not represent a special token,
|
|
||||||
# it may have different token IDs in the prompt, because
|
|
||||||
# the tokens may go across the boundaries of the search text.
|
|
||||||
# ----
|
|
||||||
# e.g. when searching for "foo" in "food", if "food" itself makes
|
|
||||||
# up a token, then the token ID of "foo" will not appear at all
|
|
||||||
# ----
|
|
||||||
# Since it is inefficient to search for all possible tokenizations
|
|
||||||
# of the search text in the prompt, we instead perform string-based
|
|
||||||
# updates on the decoded token IDs, then encode them back.
|
|
||||||
if not all(
|
|
||||||
all(update_idx is not None for update_idx in update_idxs)
|
|
||||||
for update_idxs in match_result.values()
|
|
||||||
):
|
|
||||||
new_text, match_result = self._apply_text_matches(
|
|
||||||
_seq2text(tokenizer, token_ids, use_cache=False),
|
|
||||||
mm_prompt_updates,
|
|
||||||
)
|
|
||||||
|
|
||||||
new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
|
|
||||||
|
|
||||||
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
|
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
|
||||||
for modality, update_idxs in match_result.items():
|
for modality, update_idxs in match_result.items():
|
||||||
for item_idx, update_idx in enumerate(update_idxs):
|
for item_idx, update_idx in enumerate(update_idxs):
|
||||||
@@ -1735,9 +1780,54 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
|||||||
[mm_prompt_updates[modality][item_idx][update_idx]]
|
[mm_prompt_updates[modality][item_idx][update_idx]]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
return dict(matched_updates)
|
||||||
|
|
||||||
|
def _apply_prompt_updates(
|
||||||
|
self,
|
||||||
|
token_ids: list[int],
|
||||||
|
mm_prompt_updates: MultiModalPromptUpdates,
|
||||||
|
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||||
|
"""Apply multi-modal prompt updates to token IDs."""
|
||||||
|
tokenizer = self.info.get_tokenizer()
|
||||||
|
|
||||||
|
new_token_ids, match_result, placeholders = (
|
||||||
|
self._apply_token_matches_with_placeholders(
|
||||||
|
token_ids,
|
||||||
|
mm_prompt_updates,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if all(
|
||||||
|
all(update_idx is not None for update_idx in update_idxs)
|
||||||
|
for update_idxs in match_result.values()
|
||||||
|
):
|
||||||
|
placeholders = {
|
||||||
|
modality: modality_placeholders
|
||||||
|
for modality, modality_placeholders in placeholders.items()
|
||||||
|
if modality_placeholders
|
||||||
|
}
|
||||||
|
return new_token_ids, placeholders
|
||||||
|
|
||||||
|
# If the search text does not represent a special token,
|
||||||
|
# it may have different token IDs in the prompt, because
|
||||||
|
# the tokens may go across the boundaries of the search text.
|
||||||
|
# ----
|
||||||
|
# e.g. when searching for "foo" in "food", if "food" itself makes
|
||||||
|
# up a token, then the token ID of "foo" will not appear at all
|
||||||
|
# ----
|
||||||
|
# Since it is inefficient to search for all possible tokenizations
|
||||||
|
# of the search text in the prompt, we instead perform string-based
|
||||||
|
# updates on the decoded token IDs, then encode them back.
|
||||||
|
new_text, match_result = self._apply_text_matches(
|
||||||
|
_seq2text(tokenizer, token_ids, use_cache=False),
|
||||||
|
mm_prompt_updates,
|
||||||
|
)
|
||||||
|
|
||||||
|
new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
|
||||||
|
|
||||||
placeholders = self._find_mm_placeholders(
|
placeholders = self._find_mm_placeholders(
|
||||||
new_token_ids,
|
new_token_ids,
|
||||||
dict(matched_updates),
|
self._matched_updates_from_result(mm_prompt_updates, match_result),
|
||||||
)
|
)
|
||||||
|
|
||||||
return new_token_ids, placeholders
|
return new_token_ids, placeholders
|
||||||
|
|||||||
Reference in New Issue
Block a user