[VLM] Simplify post-processing of replacement info (#12269)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -35,7 +35,7 @@ def _test_processing_correctness(
|
||||
task="auto",
|
||||
tokenizer=model_id,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=True,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
seed=0,
|
||||
dtype="float16",
|
||||
revision=None,
|
||||
|
||||
@@ -261,7 +261,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
trust_remote_code=True),
|
||||
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-Audio-7B-Instruct"), # noqa: E501
|
||||
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
|
||||
"UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_3"),
|
||||
"UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_3",
|
||||
trust_remote_code=True),
|
||||
# [Encoder-decoder]
|
||||
"MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
|
||||
"WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"), # noqa: E501
|
||||
|
||||
@@ -7,12 +7,16 @@ import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.processing import (PlaceholderInfo, PromptReplacement,
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
from vllm.multimodal.processing import (PlaceholderFeaturesInfo,
|
||||
PromptReplacement,
|
||||
find_mm_placeholders,
|
||||
find_text_matches, find_token_matches,
|
||||
iter_token_matches,
|
||||
replace_text_matches,
|
||||
replace_token_matches)
|
||||
# yapf: enable
|
||||
from vllm.multimodal.profiling import MultiModalProfiler
|
||||
from vllm.multimodal.utils import cached_get_tokenizer
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
@@ -433,19 +437,19 @@ def test_find_replace_tokens(
|
||||
[1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
|
||||
{
|
||||
"pattern_1": [
|
||||
PlaceholderInfo(
|
||||
PlaceholderFeaturesInfo(
|
||||
modality="pattern_1",
|
||||
item_idx=0,
|
||||
start_idx=6,
|
||||
replacement=[32000, 32000],
|
||||
tokens=[32000, 32000],
|
||||
),
|
||||
],
|
||||
"pattern_4": [
|
||||
PlaceholderInfo(
|
||||
PlaceholderFeaturesInfo(
|
||||
modality="pattern_4",
|
||||
item_idx=0,
|
||||
start_idx=3,
|
||||
replacement=[32000],
|
||||
tokens=[32000],
|
||||
),
|
||||
],
|
||||
}
|
||||
@@ -455,25 +459,25 @@ def test_find_replace_tokens(
|
||||
[1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],
|
||||
{
|
||||
"pattern_1": [
|
||||
PlaceholderInfo(
|
||||
PlaceholderFeaturesInfo(
|
||||
modality="pattern_1",
|
||||
item_idx=0,
|
||||
start_idx=1,
|
||||
replacement=[32000, 32000],
|
||||
tokens=[32000, 32000],
|
||||
),
|
||||
PlaceholderInfo(
|
||||
PlaceholderFeaturesInfo(
|
||||
modality="pattern_1",
|
||||
item_idx=1,
|
||||
start_idx=5,
|
||||
replacement=[32000, 32000],
|
||||
tokens=[32000, 32000],
|
||||
),
|
||||
],
|
||||
"pattern_3": [
|
||||
PlaceholderInfo(
|
||||
PlaceholderFeaturesInfo(
|
||||
modality="pattern_3",
|
||||
item_idx=0,
|
||||
start_idx=7,
|
||||
replacement=[1550, 918, 1550],
|
||||
tokens=[1550, 918, 1550],
|
||||
),
|
||||
],
|
||||
# No match for pattern_4 as it has lower priority than pattern_1
|
||||
@@ -483,33 +487,33 @@ def test_find_replace_tokens(
|
||||
[1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
|
||||
{
|
||||
"pattern_1": [
|
||||
PlaceholderInfo(
|
||||
PlaceholderFeaturesInfo(
|
||||
modality="pattern_1",
|
||||
item_idx=0,
|
||||
start_idx=1,
|
||||
replacement=[32000, 32000],
|
||||
tokens=[32000, 32000],
|
||||
),
|
||||
PlaceholderInfo(
|
||||
PlaceholderFeaturesInfo(
|
||||
modality="pattern_1",
|
||||
item_idx=1,
|
||||
start_idx=3,
|
||||
replacement=[32000, 32000],
|
||||
tokens=[32000, 32000],
|
||||
),
|
||||
],
|
||||
"pattern_4": [
|
||||
PlaceholderInfo(
|
||||
PlaceholderFeaturesInfo(
|
||||
modality="pattern_4",
|
||||
item_idx=0,
|
||||
start_idx=5,
|
||||
replacement=[32000],
|
||||
tokens=[32000],
|
||||
),
|
||||
],
|
||||
"pattern_3": [
|
||||
PlaceholderInfo(
|
||||
PlaceholderFeaturesInfo(
|
||||
modality="pattern_3",
|
||||
item_idx=0,
|
||||
start_idx=6,
|
||||
replacement=[1550, 918, 1550],
|
||||
tokens=[1550, 918, 1550],
|
||||
),
|
||||
],
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user