[Core][VLM] Add precise multi-modal placeholder tracking (#8346)
Signed-off-by: Peter Salas <peter@fixie.ai>
This commit is contained in:
@@ -5,8 +5,8 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.inputs import DecoderOnlyInputs, InputContext, token_inputs
|
||||
from vllm.inputs.registry import InputRegistry
|
||||
from vllm.inputs import (DecoderOnlyInputs, DummyData, InputContext,
|
||||
InputRegistry, token_inputs)
|
||||
from vllm.multimodal import MultiModalRegistry
|
||||
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
|
||||
|
||||
@@ -56,7 +56,7 @@ def use_dummy_data_mock():
|
||||
num_crops=DEFAULT_NUM_CROPS):
|
||||
seq_data = SequenceData(
|
||||
array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * num_crops))
|
||||
return seq_data, None
|
||||
return DummyData(seq_data, None)
|
||||
|
||||
with patch(
|
||||
"vllm.inputs.registry.InputRegistry._default_dummy_data_factory",
|
||||
@@ -177,9 +177,9 @@ def test_dummy_data_kwarg_overrides(use_dummy_data_mock, num_crops):
|
||||
# NOTE: seq_len is thrown away here since this will leverage the
|
||||
# default dummy data factory that we have patched in, whose seq
|
||||
# len is solely dependent on the value of the mm_processor_kwargs.
|
||||
seq_data, _ = dummy_registry.dummy_data_for_profiling(
|
||||
dummy_data = dummy_registry.dummy_data_for_profiling(
|
||||
ctx.model_config, seq_len=-1, mm_registry=mm_registry)
|
||||
assert len(seq_data.prompt_token_ids) == expected_seq_count
|
||||
assert len(dummy_data.seq_data.prompt_token_ids) == expected_seq_count
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -206,9 +206,9 @@ def test_dummy_data_with_sad_kwarg_overrides(use_dummy_data_mock,
|
||||
# NOTE: seq_len is thrown away here since this will leverage the
|
||||
# default dummy data factory that we have patched in, whose seq
|
||||
# len is solely dependent on the value of the mm_processor_kwargs.
|
||||
seq_data, _ = dummy_registry.dummy_data_for_profiling(
|
||||
dummy_data = dummy_registry.dummy_data_for_profiling(
|
||||
ctx.model_config, seq_len=-1, mm_registry=mm_registry)
|
||||
assert len(seq_data.prompt_token_ids) == DEFAULT_NUM_CROPS
|
||||
assert len(dummy_data.seq_data.prompt_token_ids) == DEFAULT_NUM_CROPS
|
||||
|
||||
|
||||
### Test overrides for the max token count per multimodal instance
|
||||
|
||||
@@ -92,18 +92,50 @@ def test_repeat_and_pad_placeholder_tokens(model):
|
||||
tokenizer = AutoTokenizer.from_pretrained(model)
|
||||
|
||||
test_cases = [
|
||||
("<image>", 2, "<image><image>", [32000, 32000]),
|
||||
("<image><image>", 2, "<image><image><image>", [32000, 32000, 32000]),
|
||||
("<image><image>", [3, 2], "<image><image><image><image><image>",
|
||||
[32000, 32000, 32000, 32000, 32000]),
|
||||
("Image:<image>Image:<image>!", [3, 2],
|
||||
"Image:<image><image><image>Image:<image><image>!",
|
||||
[9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918]),
|
||||
("<image>", [3, 2], "<image><image><image>", [32000, 32000, 32000]),
|
||||
]
|
||||
(
|
||||
"<image>",
|
||||
2,
|
||||
"<image><image>",
|
||||
[32000, 32000],
|
||||
[{ "offset": 0, "length": 2 }],
|
||||
),
|
||||
(
|
||||
"<image><image>",
|
||||
2,
|
||||
"<image><image><image>",
|
||||
[32000, 32000, 32000],
|
||||
[{ "offset": 0, "length": 2 }]),
|
||||
(
|
||||
"<image><image>",
|
||||
[3, 2],
|
||||
"<image><image><image><image><image>",
|
||||
[32000, 32000, 32000, 32000, 32000],
|
||||
[{ "offset": 0, "length": 3 }, { "offset": 3, "length": 2 }],
|
||||
),
|
||||
(
|
||||
"Image:<image>Image:<image>!",
|
||||
[3, 2],
|
||||
"Image:<image><image><image>Image:<image><image>!",
|
||||
[9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
|
||||
[{ "offset": 2, "length": 3 }, { "offset": 7, "length": 2 }],
|
||||
),
|
||||
(
|
||||
"<image>",
|
||||
[3, 2],
|
||||
"<image><image><image>",
|
||||
[32000, 32000, 32000],
|
||||
[{ "offset": 0, "length": 3 }],
|
||||
),
|
||||
] # yapf: disable
|
||||
|
||||
for prompt, repeat_count, expected_prompt, expected_token_ids in test_cases:
|
||||
new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
|
||||
for (
|
||||
prompt,
|
||||
repeat_count,
|
||||
expected_prompt,
|
||||
expected_token_ids,
|
||||
expected_ranges,
|
||||
) in test_cases:
|
||||
new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
|
||||
tokenizer=tokenizer,
|
||||
prompt=prompt,
|
||||
prompt_token_ids=tokenizer.encode(prompt,
|
||||
@@ -113,3 +145,4 @@ def test_repeat_and_pad_placeholder_tokens(model):
|
||||
)
|
||||
assert new_prompt == expected_prompt
|
||||
assert new_token_ids == expected_token_ids
|
||||
assert ranges == expected_ranges
|
||||
|
||||
Reference in New Issue
Block a user