Remove all references to yapf as it's no longer used (#26251)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -8,16 +8,11 @@ import torch
|
||||
import vllm.envs as envs
|
||||
from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor
|
||||
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
from vllm.compilation.activation_quant_fusion import (
|
||||
FUSED_OPS,
|
||||
SILU_MUL_OP,
|
||||
ActivationQuantFusionPass,
|
||||
)
|
||||
|
||||
# yapf: enable
|
||||
from vllm.compilation.fusion import QUANT_OPS
|
||||
from vllm.compilation.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.post_cleanup import PostCleanupPass
|
||||
|
||||
@@ -107,10 +107,8 @@ class EPTestSettings:
|
||||
# NOTE: You can adjust tp_base locally to fit the model in GPU
|
||||
# The values displayed here are only a rough indicator of the size of the model
|
||||
|
||||
# yapf: disable
|
||||
TEST_MODELS = {
|
||||
"deepseek-ai/DeepSeek-V2-Lite-Chat": EPTestSettings.fast(
|
||||
trust_remote_code=True),
|
||||
"deepseek-ai/DeepSeek-V2-Lite-Chat": EPTestSettings.fast(trust_remote_code=True),
|
||||
"mistralai/Mixtral-8x7B-Instruct-v0.1": EPTestSettings.fast(tp_base=4),
|
||||
}
|
||||
|
||||
@@ -192,22 +190,24 @@ def _compare_tp(
|
||||
]
|
||||
|
||||
try:
|
||||
compare_two_settings(model_name,
|
||||
ep_args,
|
||||
tp_args,
|
||||
ep_env,
|
||||
tp_env,
|
||||
method=method,
|
||||
max_wait_seconds=360)
|
||||
compare_two_settings(
|
||||
model_name,
|
||||
ep_args,
|
||||
tp_args,
|
||||
ep_env,
|
||||
tp_env,
|
||||
method=method,
|
||||
max_wait_seconds=360,
|
||||
)
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_name", "parallel_setup", "distributed_backend", "runner",
|
||||
"test_options"),
|
||||
("model_name", "parallel_setup", "distributed_backend", "runner", "test_options"),
|
||||
[
|
||||
params for model_name, settings in TEST_MODELS.items()
|
||||
params
|
||||
for model_name, settings in TEST_MODELS.items()
|
||||
for params in settings.iter_params(model_name)
|
||||
],
|
||||
)
|
||||
@@ -220,10 +220,12 @@ def test_ep(
|
||||
test_options: EPTestOptions,
|
||||
num_gpus_available,
|
||||
):
|
||||
_compare_tp(model_name,
|
||||
parallel_setup,
|
||||
distributed_backend,
|
||||
runner,
|
||||
test_options,
|
||||
num_gpus_available,
|
||||
method="generate")
|
||||
_compare_tp(
|
||||
model_name,
|
||||
parallel_setup,
|
||||
distributed_backend,
|
||||
runner,
|
||||
test_options,
|
||||
num_gpus_available,
|
||||
method="generate",
|
||||
)
|
||||
|
||||
@@ -100,7 +100,6 @@ class PPTestSettings:
|
||||
# NOTE: You can adjust tp_base and/or pp_base locally to fit the model in GPU
|
||||
# The values displayed here are only a rough indicator of the size of the model
|
||||
|
||||
# yapf: disable
|
||||
TEXT_GENERATION_MODELS = {
|
||||
# [Decoder-only]
|
||||
# Uses Llama
|
||||
@@ -150,7 +149,9 @@ TEXT_GENERATION_MODELS = {
|
||||
"adept/persimmon-8b-chat": PPTestSettings.fast(),
|
||||
"microsoft/phi-2": PPTestSettings.fast(),
|
||||
"microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(),
|
||||
"microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(multi_node_only=True, load_format="dummy"), # noqa: E501
|
||||
"microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(
|
||||
multi_node_only=True, load_format="dummy"
|
||||
), # noqa: E501
|
||||
"Qwen/Qwen-7B-Chat": PPTestSettings.fast(),
|
||||
"Qwen/Qwen2.5-0.5B-Instruct": PPTestSettings.fast(),
|
||||
"Qwen/Qwen1.5-MoE-A2.7B-Chat": PPTestSettings.fast(),
|
||||
@@ -196,7 +197,6 @@ MULTIMODAL_MODELS = {
|
||||
"Qwen/Qwen2-VL-2B-Instruct": PPTestSettings.fast(),
|
||||
"fixie-ai/ultravox-v0_5-llama-3_2-1b": PPTestSettings.fast(),
|
||||
}
|
||||
# yapf: enable
|
||||
|
||||
# NOTE: You can update this on your local machine to run specific tests
|
||||
TEST_MODELS = [
|
||||
|
||||
@@ -287,29 +287,15 @@ def test_prefix_cache_default():
|
||||
assert not engine_args.enable_prefix_caching
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(("arg", "expected", "option"), [
|
||||
(None, None, "mm-processor-kwargs"),
|
||||
("{}", {}, "mm-processor-kwargs"),
|
||||
(
|
||||
'{"num_crops": 4}',
|
||||
{
|
||||
"num_crops": 4
|
||||
},
|
||||
"mm-processor-kwargs"
|
||||
),
|
||||
(
|
||||
'{"foo": {"bar": "baz"}}',
|
||||
{
|
||||
"foo":
|
||||
{
|
||||
"bar": "baz"
|
||||
}
|
||||
},
|
||||
"mm-processor-kwargs"
|
||||
),
|
||||
])
|
||||
# yapf: enable
|
||||
@pytest.mark.parametrize(
|
||||
("arg", "expected", "option"),
|
||||
[
|
||||
(None, None, "mm-processor-kwargs"),
|
||||
("{}", {}, "mm-processor-kwargs"),
|
||||
('{"num_crops": 4}', {"num_crops": 4}, "mm-processor-kwargs"),
|
||||
('{"foo": {"bar": "baz"}}', {"foo": {"bar": "baz"}}, "mm-processor-kwargs"),
|
||||
],
|
||||
)
|
||||
def test_composite_arg_parser(arg, expected, option):
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
if arg is None:
|
||||
@@ -321,8 +307,7 @@ def test_composite_arg_parser(arg, expected, option):
|
||||
|
||||
def test_human_readable_model_len():
|
||||
# `exit_on_error` disabled to test invalid values below
|
||||
parser = EngineArgs.add_cli_args(
|
||||
FlexibleArgumentParser(exit_on_error=False))
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser(exit_on_error=False))
|
||||
|
||||
args = parser.parse_args([])
|
||||
assert args.max_model_len is None
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.assets.video import VideoAsset
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
_try_extract_ast,
|
||||
apply_mistral_chat_template,
|
||||
load_chat_template,
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_futures,
|
||||
@@ -1855,17 +1856,17 @@ def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwa
|
||||
|
||||
# NOTE: Qwen2-Audio default chat template is specially defined inside
|
||||
# processor class instead of using `tokenizer_config.json`
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("model", "expected_format"),
|
||||
[(PHI3V_MODEL_ID, "string"),
|
||||
(QWEN2VL_MODEL_ID, "openai"),
|
||||
(QWEN25VL_MODEL_ID, "openai"),
|
||||
(ULTRAVOX_MODEL_ID, "string"),
|
||||
(QWEN2AUDIO_MODEL_ID, "openai"),
|
||||
(LLAMA_GUARD_MODEL_ID, "openai")],
|
||||
[
|
||||
(PHI3V_MODEL_ID, "string"),
|
||||
(QWEN2VL_MODEL_ID, "openai"),
|
||||
(QWEN25VL_MODEL_ID, "openai"),
|
||||
(ULTRAVOX_MODEL_ID, "string"),
|
||||
(QWEN2AUDIO_MODEL_ID, "openai"),
|
||||
(LLAMA_GUARD_MODEL_ID, "openai"),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_resolve_content_format_hf_defined(model, expected_format):
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
@@ -1879,7 +1880,8 @@ def test_resolve_content_format_hf_defined(model, expected_format):
|
||||
hf_overrides=model_info.hf_overrides,
|
||||
skip_tokenizer_init=model_info.skip_tokenizer_init,
|
||||
enforce_eager=model_info.enforce_eager,
|
||||
dtype=model_info.dtype)
|
||||
dtype=model_info.dtype,
|
||||
)
|
||||
|
||||
tokenizer = get_tokenizer(
|
||||
model,
|
||||
@@ -1911,18 +1913,18 @@ def test_resolve_content_format_hf_defined(model, expected_format):
|
||||
assert resolved_format == expected_format
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("model", "expected_format"),
|
||||
[("Salesforce/blip2-opt-2.7b", "string"),
|
||||
("facebook/chameleon-7b", "string"),
|
||||
("deepseek-ai/deepseek-vl2-tiny", "string"),
|
||||
("adept/fuyu-8b", "string"),
|
||||
("google/paligemma-3b-mix-224", "string"),
|
||||
("Qwen/Qwen-VL", "string"),
|
||||
("Qwen/Qwen-VL-Chat", "string")],
|
||||
[
|
||||
("Salesforce/blip2-opt-2.7b", "string"),
|
||||
("facebook/chameleon-7b", "string"),
|
||||
("deepseek-ai/deepseek-vl2-tiny", "string"),
|
||||
("adept/fuyu-8b", "string"),
|
||||
("google/paligemma-3b-mix-224", "string"),
|
||||
("Qwen/Qwen-VL", "string"),
|
||||
("Qwen/Qwen-VL-Chat", "string"),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_resolve_content_format_fallbacks(model, expected_format):
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
@@ -1936,7 +1938,8 @@ def test_resolve_content_format_fallbacks(model, expected_format):
|
||||
hf_overrides=model_info.hf_overrides,
|
||||
skip_tokenizer_init=model_info.skip_tokenizer_init,
|
||||
enforce_eager=model_info.enforce_eager,
|
||||
dtype=model_info.dtype)
|
||||
dtype=model_info.dtype,
|
||||
)
|
||||
|
||||
tokenizer = get_tokenizer(
|
||||
model_config.tokenizer,
|
||||
@@ -1968,30 +1971,30 @@ def test_resolve_content_format_fallbacks(model, expected_format):
|
||||
assert resolved_format == expected_format
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("template_path", "expected_format"),
|
||||
[("template_alpaca.jinja", "string"),
|
||||
("template_baichuan.jinja", "string"),
|
||||
("template_chatglm.jinja", "string"),
|
||||
("template_chatglm2.jinja", "string"),
|
||||
("template_chatml.jinja", "string"),
|
||||
("template_dse_qwen2_vl.jinja", "openai"),
|
||||
("template_falcon_180b.jinja", "string"),
|
||||
("template_falcon.jinja", "string"),
|
||||
("template_inkbot.jinja", "string"),
|
||||
("template_teleflm.jinja", "string"),
|
||||
("template_vlm2vec_phi3v.jinja", "openai"),
|
||||
("template_vlm2vec_qwen2vl.jinja", "openai"),
|
||||
("tool_chat_template_granite_20b_fc.jinja", "string"),
|
||||
("tool_chat_template_hermes.jinja", "string"),
|
||||
("tool_chat_template_internlm2_tool.jinja", "string"),
|
||||
("tool_chat_template_llama3.1_json.jinja", "openai"),
|
||||
("tool_chat_template_llama3.2_json.jinja", "openai"),
|
||||
("tool_chat_template_mistral_parallel.jinja", "string"),
|
||||
("tool_chat_template_mistral.jinja", "string")],
|
||||
[
|
||||
("template_alpaca.jinja", "string"),
|
||||
("template_baichuan.jinja", "string"),
|
||||
("template_chatglm.jinja", "string"),
|
||||
("template_chatglm2.jinja", "string"),
|
||||
("template_chatml.jinja", "string"),
|
||||
("template_dse_qwen2_vl.jinja", "openai"),
|
||||
("template_falcon_180b.jinja", "string"),
|
||||
("template_falcon.jinja", "string"),
|
||||
("template_inkbot.jinja", "string"),
|
||||
("template_teleflm.jinja", "string"),
|
||||
("template_vlm2vec_phi3v.jinja", "openai"),
|
||||
("template_vlm2vec_qwen2vl.jinja", "openai"),
|
||||
("tool_chat_template_granite_20b_fc.jinja", "string"),
|
||||
("tool_chat_template_hermes.jinja", "string"),
|
||||
("tool_chat_template_internlm2_tool.jinja", "string"),
|
||||
("tool_chat_template_llama3.1_json.jinja", "openai"),
|
||||
("tool_chat_template_llama3.2_json.jinja", "openai"),
|
||||
("tool_chat_template_mistral_parallel.jinja", "string"),
|
||||
("tool_chat_template_mistral.jinja", "string"),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_resolve_content_format_examples(template_path, expected_format):
|
||||
model_config = ModelConfig(
|
||||
PHI3V_MODEL_ID, # Dummy
|
||||
@@ -2024,40 +2027,34 @@ def test_resolve_content_format_examples(template_path, expected_format):
|
||||
assert resolved_format == expected_format
|
||||
|
||||
|
||||
def test_parse_chat_messages_include_thinking_chunk(mistral_model_config,
|
||||
mistral_tokenizer):
|
||||
messages = [{
|
||||
"role":
|
||||
"system",
|
||||
"content": [{
|
||||
"type": "text",
|
||||
"text": "You are a helpful assistant."
|
||||
}, {
|
||||
"type":
|
||||
"thinking",
|
||||
"closed":
|
||||
True,
|
||||
"thinking":
|
||||
"Only return the answer when you are confident."
|
||||
}]
|
||||
}, {
|
||||
"role": "user",
|
||||
"content": "What is 2+2?"
|
||||
}, {
|
||||
"role":
|
||||
"assistant",
|
||||
"content": [{
|
||||
"type": "text",
|
||||
"text": "Let me think about it."
|
||||
}, {
|
||||
"type": "thinking",
|
||||
"closed": True,
|
||||
"thinking": "2+2 = 4"
|
||||
}, {
|
||||
"type": "text",
|
||||
"text": "The answer is 4.",
|
||||
}],
|
||||
}]
|
||||
def test_parse_chat_messages_include_thinking_chunk(
|
||||
mistral_model_config, mistral_tokenizer
|
||||
):
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are a helpful assistant."},
|
||||
{
|
||||
"type": "thinking",
|
||||
"closed": True,
|
||||
"thinking": "Only return the answer when you are confident.",
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "What is 2+2?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Let me think about it."},
|
||||
{"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "The answer is 4.",
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
conversation_with_thinking, _, _ = parse_chat_messages(
|
||||
messages,
|
||||
@@ -2066,122 +2063,105 @@ def test_parse_chat_messages_include_thinking_chunk(mistral_model_config,
|
||||
content_format="openai",
|
||||
)
|
||||
|
||||
expected_conversation = [{
|
||||
"role":
|
||||
"system",
|
||||
"content": [{
|
||||
"type": "text",
|
||||
"text": "You are a helpful assistant."
|
||||
}, {
|
||||
"type": "text",
|
||||
"text": "Only return the answer when you are confident."
|
||||
}],
|
||||
}, {
|
||||
"role":
|
||||
"user",
|
||||
"content": [{
|
||||
"type": "text",
|
||||
"text": "What is 2+2?"
|
||||
}],
|
||||
}, {
|
||||
"role":
|
||||
"assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Let me think about it."
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "2+2 = 4"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "The answer is 4."
|
||||
},
|
||||
]
|
||||
}]
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are a helpful assistant."},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Only return the answer when you are confident.",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "What is 2+2?"}],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Let me think about it."},
|
||||
{"type": "text", "text": "2+2 = 4"},
|
||||
{"type": "text", "text": "The answer is 4."},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
assert conversation_with_thinking == expected_conversation
|
||||
|
||||
|
||||
def test_apply_mistral_chat_template_thinking_chunk():
|
||||
# Moved import here to avoid yapf and isort conflicts
|
||||
from vllm.entrypoints.chat_utils import apply_mistral_chat_template
|
||||
messages = [{
|
||||
"role":
|
||||
"system",
|
||||
"content": [{
|
||||
"type": "text",
|
||||
"text": "You are a helpful assistant."
|
||||
}, {
|
||||
"type":
|
||||
"thinking",
|
||||
"closed":
|
||||
True,
|
||||
"thinking":
|
||||
"Only return the answer when you are confident."
|
||||
}]
|
||||
}, {
|
||||
"role": "user",
|
||||
"content": "What is 2+2?"
|
||||
}, {
|
||||
"role":
|
||||
"assistant",
|
||||
"content": [{
|
||||
"type": "text",
|
||||
"text": "Let me think about it."
|
||||
}, {
|
||||
"type": "thinking",
|
||||
"closed": True,
|
||||
"thinking": "2+2 = 4"
|
||||
}, {
|
||||
"type": "text",
|
||||
"text": "The answer is 4.",
|
||||
}],
|
||||
}, {
|
||||
"role": "user",
|
||||
"content": "Thanks, what is 3+3?"
|
||||
}]
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are a helpful assistant."},
|
||||
{
|
||||
"type": "thinking",
|
||||
"closed": True,
|
||||
"thinking": "Only return the answer when you are confident.",
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "What is 2+2?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Let me think about it."},
|
||||
{"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "The answer is 4.",
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "Thanks, what is 3+3?"},
|
||||
]
|
||||
|
||||
# TODO(Julien): upon model release change to a tokenizer already configured.
|
||||
# =================================================================
|
||||
mistral_tokenizer = MistralTokenizer.from_pretrained(
|
||||
"mistralai/Devstral-Small-2507")
|
||||
"mistralai/Devstral-Small-2507"
|
||||
)
|
||||
assert isinstance(mistral_tokenizer.tokenizer, Tekkenizer)
|
||||
# Add think special tokens to the tokenizer
|
||||
mistral_tokenizer.tokenizer._all_special_tokens[35] = SpecialTokenInfo(
|
||||
rank=35, is_control=True, token_str=SpecialTokens.begin_think.value)
|
||||
rank=35, is_control=True, token_str=SpecialTokens.begin_think.value
|
||||
)
|
||||
mistral_tokenizer.tokenizer._all_special_tokens[36] = SpecialTokenInfo(
|
||||
rank=36, is_control=True, token_str=SpecialTokens.end_think.value)
|
||||
rank=36, is_control=True, token_str=SpecialTokens.end_think.value
|
||||
)
|
||||
mistral_tokenizer.tokenizer._special_tokens_reverse_vocab = {
|
||||
k: v
|
||||
for k, v in
|
||||
mistral_tokenizer.tokenizer._special_tokens_reverse_vocab.items()
|
||||
for k, v in mistral_tokenizer.tokenizer._special_tokens_reverse_vocab.items()
|
||||
if v not in {35, 36}
|
||||
}
|
||||
mistral_tokenizer.tokenizer._special_tokens_reverse_vocab[
|
||||
SpecialTokens.begin_think.value] = 35
|
||||
SpecialTokens.begin_think.value
|
||||
] = 35
|
||||
mistral_tokenizer.tokenizer._special_tokens_reverse_vocab[
|
||||
SpecialTokens.end_think.value] = 36
|
||||
SpecialTokens.end_think.value
|
||||
] = 36
|
||||
mistral_tokenizer.instruct.BEGIN_THINK = 35
|
||||
mistral_tokenizer.instruct.END_THINK = 36
|
||||
# =================================================================
|
||||
|
||||
tokens_ids = apply_mistral_chat_template(mistral_tokenizer,
|
||||
messages,
|
||||
chat_template=None,
|
||||
tools=None)
|
||||
tokens_ids = apply_mistral_chat_template(
|
||||
mistral_tokenizer, messages, chat_template=None, tools=None
|
||||
)
|
||||
|
||||
string_tokens = mistral_tokenizer.mistral.decode(
|
||||
tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP)
|
||||
tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP
|
||||
)
|
||||
|
||||
expected_tokens = (
|
||||
r"<s>[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the"
|
||||
r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]"
|
||||
r"[INST]What is 2+2?[/INST]"
|
||||
r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4.</s>"
|
||||
r"[INST]Thanks, what is 3+3?[/INST]")
|
||||
r"[INST]Thanks, what is 3+3?[/INST]"
|
||||
)
|
||||
|
||||
assert string_tokens == expected_tokens
|
||||
|
||||
@@ -2192,37 +2172,32 @@ def test_parse_chat_messages_single_empty_audio_with_uuid(
|
||||
):
|
||||
audio_uuid = "abcd"
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[{
|
||||
"role":
|
||||
"user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {},
|
||||
"uuid": audio_uuid,
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What does the audio say?"
|
||||
},
|
||||
],
|
||||
}],
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {},
|
||||
"uuid": audio_uuid,
|
||||
},
|
||||
{"type": "text", "text": "What does the audio say?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen2_audio_model_config,
|
||||
qwen2_audio_tokenizer,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
assert conversation == [{
|
||||
"role":
|
||||
"user",
|
||||
"content":
|
||||
"Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the audio say?"
|
||||
}]
|
||||
assert conversation == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the audio say?",
|
||||
}
|
||||
]
|
||||
_assert_mm_data_inputs(mm_data, {"audio": 1})
|
||||
_assert_mm_uuids(mm_uuids,
|
||||
1,
|
||||
modality="audio",
|
||||
expected_uuids=[audio_uuid])
|
||||
_assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -2232,34 +2207,29 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
|
||||
):
|
||||
audio_uuid = "abcd"
|
||||
conversation, mm_future, mm_uuids = parse_chat_messages_futures(
|
||||
[{
|
||||
"role":
|
||||
"user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {},
|
||||
"uuid": audio_uuid,
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What does the audio say?"
|
||||
},
|
||||
],
|
||||
}],
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {},
|
||||
"uuid": audio_uuid,
|
||||
},
|
||||
{"type": "text", "text": "What does the audio say?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen2_audio_model_config,
|
||||
qwen2_audio_tokenizer,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
assert conversation == [{
|
||||
"role":
|
||||
"user",
|
||||
"content":
|
||||
"Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the audio say?"
|
||||
}]
|
||||
assert conversation == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the audio say?",
|
||||
}
|
||||
]
|
||||
_assert_mm_data_inputs(await mm_future, {"audio": 1})
|
||||
_assert_mm_uuids(mm_uuids,
|
||||
1,
|
||||
modality="audio",
|
||||
expected_uuids=[audio_uuid])
|
||||
_assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
|
||||
|
||||
@@ -12,9 +12,6 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.config.lora import LoRAConfig
|
||||
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
from vllm.lora.layers import (
|
||||
BaseLayerWithLoRA,
|
||||
ColumnParallelLinearWithLoRA,
|
||||
@@ -32,8 +29,6 @@ from vllm.lora.layers import (
|
||||
RowParallelLinearWithShardedLoRA,
|
||||
VocabParallelEmbeddingWithLoRA,
|
||||
)
|
||||
|
||||
# yapf: enable
|
||||
from vllm.lora.models import LoRALayerWeights, PackedLoRALayerWeights
|
||||
from vllm.lora.punica_wrapper import get_punica_wrapper
|
||||
from vllm.model_executor.layers.linear import (
|
||||
|
||||
@@ -17,8 +17,6 @@ import vllm.model_executor.model_loader.tensorizer
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
# yapf: disable
|
||||
from vllm.model_executor.model_loader.tensorizer import (
|
||||
TensorizerConfig,
|
||||
TensorSerializer,
|
||||
@@ -29,8 +27,6 @@ from vllm.model_executor.model_loader.tensorizer import (
|
||||
from vllm.model_executor.model_loader.tensorizer_loader import (
|
||||
BLACKLISTED_TENSORIZER_ARGS,
|
||||
)
|
||||
|
||||
# yapf: enable
|
||||
from vllm.utils import PlaceholderModule
|
||||
|
||||
from .conftest import DummyExecutor, assert_from_collective_rpc
|
||||
|
||||
@@ -45,18 +45,17 @@ from .vlm_utils.types import (
|
||||
if current_platform.is_rocm():
|
||||
os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
|
||||
|
||||
# yapf: disable
|
||||
COMMON_BROADCAST_SETTINGS = {
|
||||
"test_type": VLMTestType.IMAGE,
|
||||
"dtype": "half",
|
||||
"max_tokens": 5,
|
||||
"tensor_parallel_size": 2,
|
||||
"hf_model_kwargs": {"device_map": "auto"},
|
||||
"image_size_factors": [(.25, 0.5, 1.0)],
|
||||
"image_size_factors": [(0.25, 0.5, 1.0)],
|
||||
"distributed_executor_backend": (
|
||||
"ray",
|
||||
"mp",
|
||||
)
|
||||
),
|
||||
}
|
||||
|
||||
### Test configuration for specific models
|
||||
@@ -96,22 +95,20 @@ VLM_TEST_SETTINGS = {
|
||||
#### Core tests to always run in the CI
|
||||
"llava": VLMTestInfo(
|
||||
models=["llava-hf/llava-1.5-7b-hf"],
|
||||
test_type=(
|
||||
VLMTestType.EMBEDDING,
|
||||
VLMTestType.IMAGE,
|
||||
VLMTestType.CUSTOM_INPUTS
|
||||
),
|
||||
test_type=(VLMTestType.EMBEDDING, VLMTestType.IMAGE, VLMTestType.CUSTOM_INPUTS),
|
||||
prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
|
||||
convert_assets_to_embeddings=model_utils.get_llava_embeddings,
|
||||
max_model_len=4096,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
|
||||
custom_test_opts=[CustomTestOptions(
|
||||
inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
|
||||
formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:"
|
||||
),
|
||||
limit_mm_per_prompt={"image": 4},
|
||||
)],
|
||||
custom_test_opts=[
|
||||
CustomTestOptions(
|
||||
inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
|
||||
formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:"
|
||||
),
|
||||
limit_mm_per_prompt={"image": 4},
|
||||
)
|
||||
],
|
||||
# TODO: Revert to "auto" when CPU backend can use torch > 2.6
|
||||
dtype="bfloat16" if current_platform.is_cpu() else "auto",
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
@@ -120,27 +117,27 @@ VLM_TEST_SETTINGS = {
|
||||
models=["google/paligemma-3b-mix-224"],
|
||||
test_type=VLMTestType.IMAGE,
|
||||
prompt_formatter=identity,
|
||||
img_idx_to_prompt = lambda idx: "",
|
||||
img_idx_to_prompt=lambda idx: "",
|
||||
# Paligemma uses its own sample prompts because the default one fails
|
||||
single_image_prompts=IMAGE_ASSETS.prompts({
|
||||
"stop_sign": "caption es",
|
||||
"cherry_blossom": "What is in the picture?",
|
||||
}),
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "caption es",
|
||||
"cherry_blossom": "What is in the picture?",
|
||||
}
|
||||
),
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
|
||||
dtype="bfloat16",
|
||||
marks=[pytest.mark.skip(reason="vLLM does not support PrefixLM attention mask")], # noqa: E501
|
||||
marks=[
|
||||
pytest.mark.skip(reason="vLLM does not support PrefixLM attention mask")
|
||||
], # noqa: E501
|
||||
),
|
||||
"qwen2_5_vl": VLMTestInfo(
|
||||
models=["Qwen/Qwen2.5-VL-3B-Instruct"],
|
||||
test_type=(
|
||||
VLMTestType.IMAGE,
|
||||
VLMTestType.MULTI_IMAGE,
|
||||
VLMTestType.VIDEO
|
||||
),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
@@ -150,17 +147,13 @@ VLM_TEST_SETTINGS = {
|
||||
),
|
||||
"qwen2_5_omni": VLMTestInfo(
|
||||
models=["Qwen/Qwen2.5-Omni-3B"],
|
||||
test_type=(
|
||||
VLMTestType.IMAGE,
|
||||
VLMTestType.MULTI_IMAGE,
|
||||
VLMTestType.VIDEO
|
||||
),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|vision_bos|><|IMAGE|><|vision_eos|>", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<|vision_bos|><|VIDEO|><|vision_eos|>", # noqa: E501
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|vision_bos|><|IMAGE|><|vision_eos|>", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<|vision_bos|><|VIDEO|><|vision_eos|>", # noqa: E501
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
num_logprobs= 6 if current_platform.is_cpu() else 5,
|
||||
num_logprobs=6 if current_platform.is_cpu() else 5,
|
||||
auto_cls=AutoModelForTextToWaveform,
|
||||
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
|
||||
patch_hf_runner=model_utils.qwen2_5_omni_patch_hf_runner,
|
||||
@@ -168,9 +161,9 @@ VLM_TEST_SETTINGS = {
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
"ultravox": VLMTestInfo(
|
||||
models = ["fixie-ai/ultravox-v0_5-llama-3_2-1b"],
|
||||
models=["fixie-ai/ultravox-v0_5-llama-3_2-1b"],
|
||||
test_type=VLMTestType.AUDIO,
|
||||
prompt_formatter=lambda audio_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{audio_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
|
||||
prompt_formatter=lambda audio_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{audio_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
|
||||
audio_idx_to_prompt=lambda idx: "<|audio|>",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
@@ -184,9 +177,11 @@ VLM_TEST_SETTINGS = {
|
||||
"llava-onevision-transformers": VLMTestInfo(
|
||||
models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
|
||||
test_type=VLMTestType.IMAGE,
|
||||
prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
max_model_len=16384,
|
||||
hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
|
||||
hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
), # noqa: E501
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
|
||||
image_size_factors=[(0.25, 0.5, 1.0)],
|
||||
@@ -201,7 +196,7 @@ VLM_TEST_SETTINGS = {
|
||||
"idefics3-transformers": VLMTestInfo(
|
||||
models=["HuggingFaceTB/SmolVLM-256M-Instruct"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt:f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<image>",
|
||||
max_model_len=8192,
|
||||
max_num_seqs=2,
|
||||
@@ -217,8 +212,8 @@ VLM_TEST_SETTINGS = {
|
||||
"qwen2_5_vl-transformers": VLMTestInfo(
|
||||
models=["Qwen/Qwen2.5-VL-3B-Instruct"],
|
||||
test_type=VLMTestType.IMAGE,
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
@@ -228,23 +223,24 @@ VLM_TEST_SETTINGS = {
|
||||
"model_impl": "transformers",
|
||||
},
|
||||
# FIXME: Investigate mrope issue
|
||||
marks=[large_gpu_mark(min_gb=32),
|
||||
pytest.mark.skip(reason="Mrope issue")],
|
||||
marks=[large_gpu_mark(min_gb=32), pytest.mark.skip(reason="Mrope issue")],
|
||||
),
|
||||
#### Extended model tests
|
||||
"aria": VLMTestInfo(
|
||||
models=["rhymes-ai/Aria"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<fim_prefix><|img|><fim_suffix>\n",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
single_image_prompts=IMAGE_ASSETS.prompts({
|
||||
"stop_sign": "<vlm_image>Please describe the image shortly.",
|
||||
"cherry_blossom": "<vlm_image>Please infer the season with reason.", # noqa: E501
|
||||
}),
|
||||
multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<vlm_image>Please describe the image shortly.",
|
||||
"cherry_blossom": "<vlm_image>Please infer the season with reason.", # noqa: E501
|
||||
}
|
||||
),
|
||||
multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.", # noqa: E501
|
||||
stop_str=["<|im_end|>"],
|
||||
image_size_factors=[(0.10, 0.15)],
|
||||
max_tokens=64,
|
||||
@@ -253,11 +249,13 @@ VLM_TEST_SETTINGS = {
|
||||
"aya_vision": VLMTestInfo(
|
||||
models=["CohereForAI/aya-vision-8b"],
|
||||
test_type=(VLMTestType.IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts({
|
||||
"stop_sign": "<image>What's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<image>What is the season?", # noqa: E501
|
||||
}),
|
||||
prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<image>What's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<image>What is the season?", # noqa: E501
|
||||
}
|
||||
),
|
||||
multi_image_prompt="<image><image>Describe the two images in detail.", # noqa: E501
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
@@ -267,11 +265,13 @@ VLM_TEST_SETTINGS = {
|
||||
"aya_vision-multi_image": VLMTestInfo(
|
||||
models=["CohereForAI/aya-vision-8b"],
|
||||
test_type=(VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts({
|
||||
"stop_sign": "<image>What's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<image>What is the season?", # noqa: E501
|
||||
}),
|
||||
prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<image>What's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<image>What is the season?", # noqa: E501
|
||||
}
|
||||
),
|
||||
multi_image_prompt="<image><image>Describe the two images in detail.", # noqa: E501
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
@@ -297,27 +297,29 @@ VLM_TEST_SETTINGS = {
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
# For chameleon, we only compare the sequences
|
||||
vllm_output_post_proc = lambda vllm_output, model: vllm_output[:2],
|
||||
hf_output_post_proc = lambda hf_output, model: hf_output[:2],
|
||||
vllm_output_post_proc=lambda vllm_output, model: vllm_output[:2],
|
||||
hf_output_post_proc=lambda hf_output, model: hf_output[:2],
|
||||
comparator=check_outputs_equal,
|
||||
max_tokens=8,
|
||||
dtype="bfloat16",
|
||||
),
|
||||
"deepseek_vl_v2": VLMTestInfo(
|
||||
models=["Isotr0py/deepseek-vl2-tiny"], # model repo using dynamic module
|
||||
models=["Isotr0py/deepseek-vl2-tiny"], # model repo using dynamic module
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|User|>: {img_prompt}\n\n<|Assistant|>: ", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|User|>: {img_prompt}\n\n<|Assistant|>: ", # noqa: E501
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
single_image_prompts=IMAGE_ASSETS.prompts({
|
||||
"stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<image>\nPlease infer the season with reason in details.", # noqa: E501
|
||||
}),
|
||||
multi_image_prompt="image_1:<image>\nimage_2:<image>\nWhich image can we see the car and the tower?", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<image>\nPlease infer the season with reason in details.", # noqa: E501
|
||||
}
|
||||
),
|
||||
multi_image_prompt="image_1:<image>\nimage_2:<image>\nWhich image can we see the car and the tower?", # noqa: E501
|
||||
patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
|
||||
hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
|
||||
stop_str=["<|end▁of▁sentence|>", "<|begin▁of▁sentence|>"], # noqa: E501
|
||||
image_size_factors=[(), (1.0, ), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)],
|
||||
image_size_factors=[(), (1.0,), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)],
|
||||
),
|
||||
"fuyu": VLMTestInfo(
|
||||
models=["adept/fuyu-8b"],
|
||||
@@ -336,11 +338,13 @@ VLM_TEST_SETTINGS = {
|
||||
"gemma3": VLMTestInfo(
|
||||
models=["google/gemma-3-4b-it"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts({
|
||||
"stop_sign": "<start_of_image>What's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<start_of_image>What is the season?", # noqa: E501
|
||||
}),
|
||||
prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<start_of_image>What's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<start_of_image>What is the season?", # noqa: E501
|
||||
}
|
||||
),
|
||||
multi_image_prompt="<start_of_image><start_of_image>Describe the two images in detail.", # noqa: E501
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
@@ -353,10 +357,12 @@ VLM_TEST_SETTINGS = {
|
||||
models=["zai-org/glm-4v-9b"],
|
||||
test_type=VLMTestType.IMAGE,
|
||||
prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts({
|
||||
"stop_sign": "<|begin_of_image|><|endoftext|><|end_of_image|>What's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<|begin_of_image|><|endoftext|><|end_of_image|>What is the season?", # noqa: E501
|
||||
}),
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<|begin_of_image|><|endoftext|><|end_of_image|>What's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<|begin_of_image|><|endoftext|><|end_of_image|>What is the season?", # noqa: E501
|
||||
}
|
||||
),
|
||||
max_model_len=2048,
|
||||
max_num_seqs=2,
|
||||
get_stop_token_ids=lambda tok: [151329, 151336, 151338],
|
||||
@@ -372,8 +378,8 @@ VLM_TEST_SETTINGS = {
|
||||
models=["zai-org/GLM-4.1V-9B-Thinking"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>", # noqa: E501
|
||||
max_model_len=2048,
|
||||
max_num_seqs=2,
|
||||
get_stop_token_ids=lambda tok: [151329, 151336, 151338],
|
||||
@@ -390,23 +396,27 @@ VLM_TEST_SETTINGS = {
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
patch_hf_runner=model_utils.glm4_1v_patch_hf_runner,
|
||||
custom_test_opts=[CustomTestOptions(
|
||||
inputs=custom_inputs.video_with_metadata_glm4_1v(),
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
)],
|
||||
custom_test_opts=[
|
||||
CustomTestOptions(
|
||||
inputs=custom_inputs.video_with_metadata_glm4_1v(),
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
)
|
||||
],
|
||||
marks=[large_gpu_mark(min_gb=32)],
|
||||
),
|
||||
"h2ovl": VLMTestInfo(
|
||||
models = [
|
||||
models=[
|
||||
"h2oai/h2ovl-mississippi-800m",
|
||||
"h2oai/h2ovl-mississippi-2b",
|
||||
],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|prompt|>{img_prompt}<|end|><|answer|>", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts({
|
||||
"stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<image>\nWhat is the season?",
|
||||
}),
|
||||
prompt_formatter=lambda img_prompt: f"<|prompt|>{img_prompt}<|end|><|answer|>", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<image>\nWhat is the season?",
|
||||
}
|
||||
),
|
||||
multi_image_prompt="Image-1: <image>\nImage-2: <image>\nDescribe the two images in short.", # noqa: E501
|
||||
max_model_len=8192,
|
||||
use_tokenizer_eos=True,
|
||||
@@ -416,7 +426,7 @@ VLM_TEST_SETTINGS = {
|
||||
"idefics3": VLMTestInfo(
|
||||
models=["HuggingFaceTB/SmolVLM-256M-Instruct"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt:f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<image>",
|
||||
max_model_len=8192,
|
||||
max_num_seqs=2,
|
||||
@@ -431,11 +441,13 @@ VLM_TEST_SETTINGS = {
|
||||
# "OpenGVLab/Mono-InternVL-2B",
|
||||
],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts({
|
||||
"stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<image>\nWhat is the season?",
|
||||
}),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<image>\nWhat is the season?",
|
||||
}
|
||||
),
|
||||
multi_image_prompt="Image-1: <image>\nImage-2: <image>\nDescribe the two images in short.", # noqa: E501
|
||||
max_model_len=4096,
|
||||
use_tokenizer_eos=True,
|
||||
@@ -446,7 +458,7 @@ VLM_TEST_SETTINGS = {
|
||||
"OpenGVLab/InternVL3-1B",
|
||||
],
|
||||
test_type=VLMTestType.VIDEO,
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<video>",
|
||||
max_model_len=8192,
|
||||
use_tokenizer_eos=True,
|
||||
@@ -459,7 +471,7 @@ VLM_TEST_SETTINGS = {
|
||||
VLMTestType.MULTI_IMAGE,
|
||||
VLMTestType.VIDEO,
|
||||
),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<IMG_CONTEXT>",
|
||||
video_idx_to_prompt=lambda idx: "<video>",
|
||||
max_model_len=8192,
|
||||
@@ -469,7 +481,7 @@ VLM_TEST_SETTINGS = {
|
||||
"kimi_vl": VLMTestInfo(
|
||||
models=["moonshotai/Kimi-VL-A3B-Instruct"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_user|>user<|im_middle|>{img_prompt}<|im_end|><|im_assistant|>assistant<|im_middle|>", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|im_user|>user<|im_middle|>{img_prompt}<|im_end|><|im_assistant|>assistant<|im_middle|>", # noqa: E501
|
||||
img_idx_to_prompt=lambda _: "<|media_start|>image<|media_content|><|media_pad|><|media_end|>", # noqa: E501
|
||||
max_model_len=8192,
|
||||
max_num_seqs=2,
|
||||
@@ -480,11 +492,11 @@ VLM_TEST_SETTINGS = {
|
||||
),
|
||||
"llama4": VLMTestInfo(
|
||||
models=["meta-llama/Llama-4-Scout-17B-16E-Instruct"],
|
||||
prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|header_start|>user<|header_end|>\n\n{img_prompt}<|eot|><|header_start|>assistant<|header_end|>\n\n", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|header_start|>user<|header_end|>\n\n{img_prompt}<|eot|><|header_start|>assistant<|header_end|>\n\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda _: "<|image|>",
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
distributed_executor_backend="mp",
|
||||
image_size_factors=[(.25, 0.5, 1.0)],
|
||||
image_size_factors=[(0.25, 0.5, 1.0)],
|
||||
hf_model_kwargs={"device_map": "auto"},
|
||||
max_model_len=8192,
|
||||
max_num_seqs=4,
|
||||
@@ -500,28 +512,34 @@ VLM_TEST_SETTINGS = {
|
||||
max_model_len=10240,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
|
||||
custom_test_opts=[CustomTestOptions(
|
||||
inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
|
||||
formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]"
|
||||
),
|
||||
limit_mm_per_prompt={"image": 4},
|
||||
)],
|
||||
custom_test_opts=[
|
||||
CustomTestOptions(
|
||||
inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
|
||||
formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]"
|
||||
),
|
||||
limit_mm_per_prompt={"image": 4},
|
||||
)
|
||||
],
|
||||
),
|
||||
"llava_onevision": VLMTestInfo(
|
||||
models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
|
||||
test_type=VLMTestType.CUSTOM_INPUTS,
|
||||
prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
num_video_frames=16,
|
||||
max_model_len=16384,
|
||||
hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
|
||||
hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
), # noqa: E501
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
|
||||
custom_test_opts=[CustomTestOptions(
|
||||
inputs=custom_inputs.multi_video_multi_aspect_ratio_inputs(
|
||||
formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
),
|
||||
limit_mm_per_prompt={"video": 4},
|
||||
)],
|
||||
custom_test_opts=[
|
||||
CustomTestOptions(
|
||||
inputs=custom_inputs.multi_video_multi_aspect_ratio_inputs(
|
||||
formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
),
|
||||
limit_mm_per_prompt={"video": 4},
|
||||
)
|
||||
],
|
||||
),
|
||||
"llava_next_video": VLMTestInfo(
|
||||
models=["llava-hf/LLaVA-NeXT-Video-7B-hf"],
|
||||
@@ -563,7 +581,9 @@ VLM_TEST_SETTINGS = {
|
||||
img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
|
||||
get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(
|
||||
["<|im_end|>", "<|endoftext|>"]
|
||||
), # noqa: E501
|
||||
hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
|
||||
patch_hf_runner=model_utils.minicpmo_26_patch_hf_runner,
|
||||
# FIXME: https://huggingface.co/openbmb/MiniCPM-o-2_6/discussions/49
|
||||
@@ -576,13 +596,15 @@ VLM_TEST_SETTINGS = {
|
||||
img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
|
||||
get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(
|
||||
["<|im_end|>", "<|endoftext|>"]
|
||||
), # noqa: E501
|
||||
hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
|
||||
patch_hf_runner=model_utils.minicpmv_26_patch_hf_runner,
|
||||
),
|
||||
"minimax_vl_01": VLMTestInfo(
|
||||
models=["MiniMaxAI/MiniMax-VL-01"],
|
||||
prompt_formatter=lambda img_prompt: f"<beginning_of_sentence>user: {img_prompt} assistant:<end_of_sentence>", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<beginning_of_sentence>user: {img_prompt} assistant:<end_of_sentence>", # noqa: E501
|
||||
img_idx_to_prompt=lambda _: "<image>",
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
max_model_len=8192,
|
||||
@@ -604,8 +626,8 @@ VLM_TEST_SETTINGS = {
|
||||
"ovis1_6-gemma2": VLMTestInfo(
|
||||
models=["AIDC-AI/Ovis1.6-Gemma2-9B"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
dtype="half",
|
||||
@@ -617,8 +639,8 @@ VLM_TEST_SETTINGS = {
|
||||
"ovis2": VLMTestInfo(
|
||||
models=["AIDC-AI/Ovis2-1B"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
dtype="half",
|
||||
@@ -628,13 +650,9 @@ VLM_TEST_SETTINGS = {
|
||||
),
|
||||
"ovis2_5": VLMTestInfo(
|
||||
models=["AIDC-AI/Ovis2.5-2B"],
|
||||
test_type=(
|
||||
VLMTestType.IMAGE,
|
||||
VLMTestType.MULTI_IMAGE,
|
||||
VLMTestType.VIDEO
|
||||
),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<video>\n",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
@@ -646,7 +664,7 @@ VLM_TEST_SETTINGS = {
|
||||
"phi3v": VLMTestInfo(
|
||||
models=["microsoft/Phi-3.5-vision-instruct"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|end|>\n<|assistant|>\n", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|end|>\n<|assistant|>\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: f"<|image_{idx}|>\n",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
@@ -681,15 +699,11 @@ VLM_TEST_SETTINGS = {
|
||||
),
|
||||
"qwen2_vl": VLMTestInfo(
|
||||
models=["Qwen/Qwen2-VL-2B-Instruct"],
|
||||
test_type=(
|
||||
VLMTestType.IMAGE,
|
||||
VLMTestType.MULTI_IMAGE,
|
||||
VLMTestType.VIDEO
|
||||
),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
|
||||
multi_image_prompt="Picture 1: <vlm_image>\nPicture 2: <vlm_image>\nDescribe these two images with one paragraph respectively.", # noqa: E501
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
|
||||
multi_image_prompt="Picture 1: <vlm_image>\nPicture 2: <vlm_image>\nDescribe these two images with one paragraph respectively.", # noqa: E501
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
@@ -700,11 +714,13 @@ VLM_TEST_SETTINGS = {
|
||||
"skywork_r1v": VLMTestInfo(
|
||||
models=["Skywork/Skywork-R1V-38B"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|begin▁of▁sentence|><|User|>\n{img_prompt}<|Assistant|><think>\n", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts({
|
||||
"stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<image>\nWhat is the season?",
|
||||
}),
|
||||
prompt_formatter=lambda img_prompt: f"<|begin▁of▁sentence|><|User|>\n{img_prompt}<|Assistant|><think>\n", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
|
||||
"cherry_blossom": "<image>\nWhat is the season?",
|
||||
}
|
||||
),
|
||||
multi_image_prompt="<image>\n<image>\nDescribe the two images in short.", # noqa: E501
|
||||
max_model_len=4096,
|
||||
use_tokenizer_eos=True,
|
||||
@@ -737,9 +753,9 @@ VLM_TEST_SETTINGS = {
|
||||
VLMTestType.MULTI_IMAGE,
|
||||
VLMTestType.VIDEO,
|
||||
),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
@@ -752,11 +768,11 @@ VLM_TEST_SETTINGS = {
|
||||
prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
|
||||
max_model_len=4096,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_output_post_proc = lambda vllm_output, model: vllm_output[:2],
|
||||
hf_output_post_proc = lambda hf_output, model: hf_output[:2],
|
||||
vllm_output_post_proc=lambda vllm_output, model: vllm_output[:2],
|
||||
hf_output_post_proc=lambda hf_output, model: hf_output[:2],
|
||||
comparator=check_outputs_equal,
|
||||
marks=multi_gpu_marks(num_gpus=2),
|
||||
**COMMON_BROADCAST_SETTINGS # type: ignore
|
||||
**COMMON_BROADCAST_SETTINGS, # type: ignore
|
||||
),
|
||||
"llava-broadcast": VLMTestInfo(
|
||||
models=["llava-hf/llava-1.5-7b-hf"],
|
||||
@@ -765,7 +781,7 @@ VLM_TEST_SETTINGS = {
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
|
||||
marks=multi_gpu_marks(num_gpus=2),
|
||||
**COMMON_BROADCAST_SETTINGS # type: ignore
|
||||
**COMMON_BROADCAST_SETTINGS, # type: ignore
|
||||
),
|
||||
"llava_next-broadcast": VLMTestInfo(
|
||||
models=["llava-hf/llava-v1.6-mistral-7b-hf"],
|
||||
@@ -774,12 +790,12 @@ VLM_TEST_SETTINGS = {
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
|
||||
marks=multi_gpu_marks(num_gpus=2),
|
||||
**COMMON_BROADCAST_SETTINGS # type: ignore
|
||||
**COMMON_BROADCAST_SETTINGS, # type: ignore
|
||||
),
|
||||
### Custom input edge-cases for specific models
|
||||
"intern_vl-diff-patches": VLMTestInfo(
|
||||
models=["OpenGVLab/InternVL2-2B"],
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
test_type=VLMTestType.CUSTOM_INPUTS,
|
||||
max_model_len=4096,
|
||||
use_tokenizer_eos=True,
|
||||
@@ -788,7 +804,8 @@ VLM_TEST_SETTINGS = {
|
||||
CustomTestOptions(
|
||||
inputs=inp,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
) for inp in custom_inputs.different_patch_input_cases_internvl()
|
||||
)
|
||||
for inp in custom_inputs.different_patch_input_cases_internvl()
|
||||
],
|
||||
),
|
||||
"llava_onevision-multiple-images": VLMTestInfo(
|
||||
@@ -797,14 +814,18 @@ VLM_TEST_SETTINGS = {
|
||||
max_model_len=16384,
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
|
||||
hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
), # noqa: E501
|
||||
vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
|
||||
custom_test_opts=[CustomTestOptions(
|
||||
inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
|
||||
formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
),
|
||||
limit_mm_per_prompt={"image": 4},
|
||||
)],
|
||||
custom_test_opts=[
|
||||
CustomTestOptions(
|
||||
inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
|
||||
formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
),
|
||||
limit_mm_per_prompt={"image": 4},
|
||||
)
|
||||
],
|
||||
),
|
||||
# regression test for https://github.com/vllm-project/vllm/issues/15122
|
||||
"qwen2_5_vl-windows-attention": VLMTestInfo(
|
||||
@@ -814,13 +835,14 @@ VLM_TEST_SETTINGS = {
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
|
||||
custom_test_opts=[CustomTestOptions(
|
||||
inputs=custom_inputs.windows_attention_image_qwen2_5_vl(),
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)],
|
||||
custom_test_opts=[
|
||||
CustomTestOptions(
|
||||
inputs=custom_inputs.windows_attention_image_qwen2_5_vl(),
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
],
|
||||
),
|
||||
}
|
||||
# yapf: enable
|
||||
|
||||
|
||||
def _mark_splits(
|
||||
|
||||
@@ -114,7 +114,6 @@ def get_parametrized_options(
|
||||
raise ValueError("Test has type CUSTOM_INPUTS, but none given")
|
||||
iter_kwargs["custom_test_opts"] = test_info.custom_test_opts
|
||||
|
||||
# yapf: disable
|
||||
# Wrap all model cases in a pytest parameter & pass marks through
|
||||
return [
|
||||
pytest.param(
|
||||
@@ -122,10 +121,10 @@ def get_parametrized_options(
|
||||
ExpandableVLMTestArgs(
|
||||
**{k: v for k, v in zip(iter_kwargs.keys(), case)}
|
||||
),
|
||||
marks=test_info.marks if test_info.marks is not None else []
|
||||
) for case in list(itertools.product(*iter_kwargs.values()))
|
||||
marks=test_info.marks if test_info.marks is not None else [],
|
||||
)
|
||||
for case in list(itertools.product(*iter_kwargs.values()))
|
||||
]
|
||||
# yapf: enable
|
||||
|
||||
# Get a list per model type, where each entry contains a tuple of all of
|
||||
# that model type's cases, then flatten them into the top level so that
|
||||
|
||||
@@ -418,7 +418,6 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
self.image_size = self.vision_config.image_size
|
||||
|
||||
def __call__(self, text: str, images: Union[Image, list[Image]], **kwargs):
|
||||
# yapf: disable
|
||||
from vllm.model_executor.models.h2ovl import (
|
||||
IMG_CONTEXT,
|
||||
IMG_END,
|
||||
@@ -426,7 +425,6 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
image_to_pixel_values_h2ovl,
|
||||
)
|
||||
|
||||
# yapf: enable
|
||||
images = [images] if isinstance(images, Image) else images
|
||||
pixel_values = [
|
||||
image_to_pixel_values_h2ovl(
|
||||
|
||||
@@ -33,24 +33,26 @@ TEST_IMG_PLACEHOLDER = "<vlm_image>"
|
||||
TEST_VIDEO_PLACEHOLDER = "<vlm_video>"
|
||||
TEST_AUDIO_PLACEHOLDER = "<lmm_audio>"
|
||||
|
||||
# yapf: disable
|
||||
SINGLE_IMAGE_BASE_PROMPTS = IMAGE_ASSETS.prompts({
|
||||
"stop_sign": f"{TEST_IMG_PLACEHOLDER}What's the content of the image?",
|
||||
"cherry_blossom": f"{TEST_IMG_PLACEHOLDER}What is the season?",
|
||||
})
|
||||
SINGLE_AUDIO_BASE_PROMPT = AUDIO_ASSETS.prompts({
|
||||
"mary_had_lamb": f"{TEST_AUDIO_PLACEHOLDER}Transcribe this audio into English.", # noqa: E501
|
||||
"winning_call": f"{TEST_AUDIO_PLACEHOLDER}What is happening in this audio clip?", # noqa: E501
|
||||
})
|
||||
SINGLE_IMAGE_BASE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": f"{TEST_IMG_PLACEHOLDER}What's the content of the image?",
|
||||
"cherry_blossom": f"{TEST_IMG_PLACEHOLDER}What is the season?",
|
||||
}
|
||||
)
|
||||
SINGLE_AUDIO_BASE_PROMPT = AUDIO_ASSETS.prompts(
|
||||
{
|
||||
"mary_had_lamb": f"{TEST_AUDIO_PLACEHOLDER}Transcribe this audio into English.", # noqa: E501
|
||||
"winning_call": f"{TEST_AUDIO_PLACEHOLDER}What is happening in this audio clip?", # noqa: E501
|
||||
}
|
||||
)
|
||||
|
||||
MULTI_IMAGE_BASE_PROMPT = f"Image-1: {TEST_IMG_PLACEHOLDER}Image-2: {TEST_IMG_PLACEHOLDER}Describe the two images in detail.\n" # noqa: E501
|
||||
VIDEO_BASE_PROMPT = f"{TEST_VIDEO_PLACEHOLDER}Why is this video funny?"
|
||||
|
||||
|
||||
IMAGE_SIZE_FACTORS = [(), (1.0, ), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]
|
||||
EMBEDDING_SIZE_FACTORS = [(), (1.0, ), (1.0, 1.0, 1.0)]
|
||||
IMAGE_SIZE_FACTORS = [(), (1.0,), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]
|
||||
EMBEDDING_SIZE_FACTORS = [(), (1.0,), (1.0, 1.0, 1.0)]
|
||||
RunnerOutput = tuple[list[int], str, Optional[SampleLogprobs]]
|
||||
# yapf: enable
|
||||
|
||||
|
||||
class PromptWithMultiModalInput(NamedTuple):
|
||||
|
||||
@@ -322,80 +322,81 @@ def _test_processing_correctness_one(
|
||||
)
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize("model_id", [
|
||||
"rhymes-ai/Aria",
|
||||
"CohereForAI/aya-vision-8b",
|
||||
"Salesforce/blip2-opt-2.7b",
|
||||
"facebook/chameleon-7b",
|
||||
"CohereLabs/command-a-vision-07-2025",
|
||||
"deepseek-ai/deepseek-vl2-tiny",
|
||||
"baidu/ERNIE-4.5-VL-28B-A3B-PT",
|
||||
"adept/fuyu-8b",
|
||||
"google/gemma-3-4b-it",
|
||||
"google/gemma-3n-E2B-it",
|
||||
"zai-org/glm-4v-9b",
|
||||
"zai-org/GLM-4.1V-9B-Thinking",
|
||||
"zai-org/GLM-4.5V",
|
||||
"ibm-granite/granite-speech-3.3-2b",
|
||||
"h2oai/h2ovl-mississippi-800m",
|
||||
"naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
|
||||
"HuggingFaceM4/Idefics3-8B-Llama3",
|
||||
"internlm/Intern-S1",
|
||||
"OpenGVLab/InternVL2-1B",
|
||||
"OpenGVLab/InternVL3-1B",
|
||||
"OpenGVLab/InternVL3_5-1B",
|
||||
"OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview",
|
||||
"OpenGVLab/InternVL3_5-30B-A3B",
|
||||
"Kwai-Keye/Keye-VL-8B-Preview",
|
||||
"Kwai-Keye/Keye-VL-1_5-8B",
|
||||
"moonshotai/Kimi-VL-A3B-Instruct",
|
||||
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
"llava-hf/llava-1.5-7b-hf",
|
||||
"llava-hf/llava-v1.6-mistral-7b-hf",
|
||||
"llava-hf/LLaVA-NeXT-Video-7B-hf",
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
|
||||
"TIGER-Lab/Mantis-8B-siglip-llama3",
|
||||
"mispeech/midashenglm-7b",
|
||||
"openbmb/MiniCPM-Llama3-V-2_5",
|
||||
"openbmb/MiniCPM-o-2_6",
|
||||
"openbmb/MiniCPM-V-2_6",
|
||||
"MiniMaxAI/MiniMax-VL-01",
|
||||
"allenai/Molmo-7B-D-0924",
|
||||
"allenai/Molmo-7B-O-0924",
|
||||
"nvidia/NVLM-D-72B",
|
||||
"nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
|
||||
"AIDC-AI/Ovis1.6-Gemma2-9B",
|
||||
"AIDC-AI/Ovis1.6-Llama3.2-3B",
|
||||
"AIDC-AI/Ovis2-1B",
|
||||
"AIDC-AI/Ovis2.5-2B",
|
||||
"google/paligemma-3b-mix-224",
|
||||
"google/paligemma2-3b-ft-docci-448",
|
||||
"microsoft/Phi-3.5-vision-instruct",
|
||||
"microsoft/Phi-4-multimodal-instruct",
|
||||
"mistralai/Pixtral-12B-2409",
|
||||
"mistral-community/pixtral-12b",
|
||||
"Qwen/Qwen-VL-Chat",
|
||||
"Qwen/Qwen2-VL-2B-Instruct",
|
||||
"Qwen/Qwen2.5-VL-3B-Instruct",
|
||||
"Qwen/Qwen2-Audio-7B-Instruct",
|
||||
"Qwen/Qwen2.5-Omni-3B",
|
||||
"Qwen/Qwen3-VL-4B-Instruct",
|
||||
"Qwen/Qwen3-VL-30B-A3B-Instruct",
|
||||
"YannQi/R-4B",
|
||||
"Skywork/Skywork-R1V-38B",
|
||||
"HuggingFaceTB/SmolVLM2-2.2B-Instruct",
|
||||
"stepfun-ai/step3",
|
||||
"fixie-ai/ultravox-v0_5-llama-3_2-1b",
|
||||
"openai/whisper-large-v3",
|
||||
"omni-research/Tarsier-7b",
|
||||
"omni-research/Tarsier2-Recap-7b",
|
||||
"mistralai/Voxtral-Mini-3B-2507",
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
"rhymes-ai/Aria",
|
||||
"CohereForAI/aya-vision-8b",
|
||||
"Salesforce/blip2-opt-2.7b",
|
||||
"facebook/chameleon-7b",
|
||||
"CohereLabs/command-a-vision-07-2025",
|
||||
"deepseek-ai/deepseek-vl2-tiny",
|
||||
"baidu/ERNIE-4.5-VL-28B-A3B-PT",
|
||||
"adept/fuyu-8b",
|
||||
"google/gemma-3-4b-it",
|
||||
"google/gemma-3n-E2B-it",
|
||||
"zai-org/glm-4v-9b",
|
||||
"zai-org/GLM-4.1V-9B-Thinking",
|
||||
"zai-org/GLM-4.5V",
|
||||
"ibm-granite/granite-speech-3.3-2b",
|
||||
"h2oai/h2ovl-mississippi-800m",
|
||||
"naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
|
||||
"HuggingFaceM4/Idefics3-8B-Llama3",
|
||||
"internlm/Intern-S1",
|
||||
"OpenGVLab/InternVL2-1B",
|
||||
"OpenGVLab/InternVL3-1B",
|
||||
"OpenGVLab/InternVL3_5-1B",
|
||||
"OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview",
|
||||
"OpenGVLab/InternVL3_5-30B-A3B",
|
||||
"Kwai-Keye/Keye-VL-8B-Preview",
|
||||
"Kwai-Keye/Keye-VL-1_5-8B",
|
||||
"moonshotai/Kimi-VL-A3B-Instruct",
|
||||
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
"llava-hf/llava-1.5-7b-hf",
|
||||
"llava-hf/llava-v1.6-mistral-7b-hf",
|
||||
"llava-hf/LLaVA-NeXT-Video-7B-hf",
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
|
||||
"TIGER-Lab/Mantis-8B-siglip-llama3",
|
||||
"mispeech/midashenglm-7b",
|
||||
"openbmb/MiniCPM-Llama3-V-2_5",
|
||||
"openbmb/MiniCPM-o-2_6",
|
||||
"openbmb/MiniCPM-V-2_6",
|
||||
"MiniMaxAI/MiniMax-VL-01",
|
||||
"allenai/Molmo-7B-D-0924",
|
||||
"allenai/Molmo-7B-O-0924",
|
||||
"nvidia/NVLM-D-72B",
|
||||
"nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
|
||||
"AIDC-AI/Ovis1.6-Gemma2-9B",
|
||||
"AIDC-AI/Ovis1.6-Llama3.2-3B",
|
||||
"AIDC-AI/Ovis2-1B",
|
||||
"AIDC-AI/Ovis2.5-2B",
|
||||
"google/paligemma-3b-mix-224",
|
||||
"google/paligemma2-3b-ft-docci-448",
|
||||
"microsoft/Phi-3.5-vision-instruct",
|
||||
"microsoft/Phi-4-multimodal-instruct",
|
||||
"mistralai/Pixtral-12B-2409",
|
||||
"mistral-community/pixtral-12b",
|
||||
"Qwen/Qwen-VL-Chat",
|
||||
"Qwen/Qwen2-VL-2B-Instruct",
|
||||
"Qwen/Qwen2.5-VL-3B-Instruct",
|
||||
"Qwen/Qwen2-Audio-7B-Instruct",
|
||||
"Qwen/Qwen2.5-Omni-3B",
|
||||
"Qwen/Qwen3-VL-4B-Instruct",
|
||||
"Qwen/Qwen3-VL-30B-A3B-Instruct",
|
||||
"YannQi/R-4B",
|
||||
"Skywork/Skywork-R1V-38B",
|
||||
"HuggingFaceTB/SmolVLM2-2.2B-Instruct",
|
||||
"stepfun-ai/step3",
|
||||
"fixie-ai/ultravox-v0_5-llama-3_2-1b",
|
||||
"openai/whisper-large-v3",
|
||||
"omni-research/Tarsier-7b",
|
||||
"omni-research/Tarsier2-Recap-7b",
|
||||
"mistralai/Voxtral-Mini-3B-2507",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
|
||||
@pytest.mark.parametrize("num_batches", [32])
|
||||
@pytest.mark.parametrize("simplify_rate", [1.0])
|
||||
# yapf: enable
|
||||
def test_processing_correctness(
|
||||
model_id: str,
|
||||
hit_rate: float,
|
||||
|
||||
@@ -12,7 +12,6 @@ from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["HuggingFaceM4/Idefics3-8B-Llama3"])
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img"),
|
||||
[
|
||||
@@ -20,7 +19,6 @@ from ...utils import build_model_context
|
||||
({"size": {"longest_edge": 728}}, 169 * (2**2 + 1)),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
|
||||
@@ -11,7 +11,6 @@ from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["microsoft/Phi-3.5-vision-instruct"])
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img"),
|
||||
[
|
||||
@@ -21,7 +20,6 @@ from ...utils import build_model_context
|
||||
({}, 757),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
|
||||
@@ -11,7 +11,6 @@ from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["microsoft/Phi-4-multimodal-instruct"])
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img"),
|
||||
[
|
||||
@@ -21,7 +20,6 @@ from ...utils import build_model_context
|
||||
({}, 9585),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
|
||||
@@ -10,7 +10,6 @@ from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img", "expected_pixels_shape"),
|
||||
[
|
||||
@@ -18,7 +17,6 @@ from ...utils import build_model_context
|
||||
({"min_pixels": 64**2, "max_pixels": 512**2}, 330, (1320, 1176)),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
|
||||
@@ -12,7 +12,6 @@ from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["HuggingFaceTB/SmolVLM2-2.2B-Instruct"])
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img"),
|
||||
[
|
||||
@@ -20,7 +19,6 @@ from ...utils import build_model_context
|
||||
({"max_image_size": {"longest_edge": 768}}, 405),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
|
||||
@@ -7,9 +7,7 @@ from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize("model_id",
|
||||
["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||
def test_multimodal_processor(model_id):
|
||||
model_config = ModelConfig(
|
||||
model=model_id,
|
||||
@@ -18,9 +16,9 @@ def test_multimodal_processor(model_id):
|
||||
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
image_pil = ImageAsset('cherry_blossom').pil_image
|
||||
image_pil = ImageAsset("cherry_blossom").pil_image
|
||||
mm_data = {"image": image_pil}
|
||||
str_prompt = "<|im_start|>user <image>\nWhat is the content of this image?<|im_end|><|im_start|>assistant\n" # noqa: E501
|
||||
str_prompt = "<|im_start|>user <image>\nWhat is the content of this image?<|im_end|><|im_start|>assistant\n" # noqa: E501
|
||||
str_processed_inputs = mm_processor.apply(
|
||||
prompt=str_prompt,
|
||||
mm_data=mm_data,
|
||||
@@ -28,8 +26,23 @@ def test_multimodal_processor(model_id):
|
||||
)
|
||||
|
||||
ids_prompt = [
|
||||
151644, 872, 220, 151646, 198, 3838, 374, 279, 2213, 315, 419, 2168,
|
||||
30, 151645, 151644, 77091, 198
|
||||
151644,
|
||||
872,
|
||||
220,
|
||||
151646,
|
||||
198,
|
||||
3838,
|
||||
374,
|
||||
279,
|
||||
2213,
|
||||
315,
|
||||
419,
|
||||
2168,
|
||||
30,
|
||||
151645,
|
||||
151644,
|
||||
77091,
|
||||
198,
|
||||
]
|
||||
ids_processed_inputs = mm_processor.apply(
|
||||
prompt=ids_prompt,
|
||||
@@ -37,5 +50,7 @@ def test_multimodal_processor(model_id):
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert (str_processed_inputs["prompt_token_ids"]
|
||||
== ids_processed_inputs["prompt_token_ids"])
|
||||
assert (
|
||||
str_processed_inputs["prompt_token_ids"]
|
||||
== ids_processed_inputs["prompt_token_ids"]
|
||||
)
|
||||
|
||||
+648
-411
File diff suppressed because it is too large
Load Diff
@@ -71,25 +71,27 @@ def _dummy_items(
|
||||
)
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("item", "expected_size"),
|
||||
[
|
||||
(_dummy_item("a", {"a1": 100}), 100),
|
||||
(_dummy_item("a", {"a1": 100, "a2": 110}), 210),
|
||||
(_dummy_items({"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}), 460), # noqa: E501
|
||||
(_dummy_items({"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}).get_data(), 460), # noqa: E501
|
||||
(
|
||||
_dummy_items(
|
||||
{"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}
|
||||
).get_data(),
|
||||
460,
|
||||
), # noqa: E501
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_cache_item_size(item, expected_size):
|
||||
cache = MultiModalCache.get_lru_cache(2048, type(item))
|
||||
|
||||
cache[""] = item
|
||||
assert cache.currsize == expected_size
|
||||
|
||||
prompt_update = PromptInsertion("dummy", "target", "insertion") \
|
||||
.resolve(0)
|
||||
prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)
|
||||
|
||||
cache[""] = MultiModalProcessorCacheItem(item, [prompt_update])
|
||||
assert cache.currsize == expected_size
|
||||
@@ -106,9 +108,9 @@ def _create_vllm_config(
|
||||
return VllmConfig(
|
||||
model_config=ModelConfig(
|
||||
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
|
||||
mm_processor_cache_gb=mm_processor_cache_gb),
|
||||
parallel_config=ParallelConfig(
|
||||
data_parallel_size=1 if enable_ipc else 2),
|
||||
mm_processor_cache_gb=mm_processor_cache_gb,
|
||||
),
|
||||
parallel_config=ParallelConfig(data_parallel_size=1 if enable_ipc else 2),
|
||||
)
|
||||
|
||||
|
||||
@@ -124,11 +126,9 @@ def _compare_caches(
|
||||
seed: int = 0,
|
||||
):
|
||||
cache_0_p0 = processor_cache_from_config(config_0, MULTIMODAL_REGISTRY)
|
||||
cache_0_p1 = engine_receiver_cache_from_config(config_0,
|
||||
MULTIMODAL_REGISTRY)
|
||||
cache_0_p1 = engine_receiver_cache_from_config(config_0, MULTIMODAL_REGISTRY)
|
||||
cache_1_p0 = processor_cache_from_config(config_1, MULTIMODAL_REGISTRY)
|
||||
cache_1_p1 = engine_receiver_cache_from_config(config_1,
|
||||
MULTIMODAL_REGISTRY)
|
||||
cache_1_p1 = engine_receiver_cache_from_config(config_1, MULTIMODAL_REGISTRY)
|
||||
|
||||
cache_size_gb = max(
|
||||
config_0.model_config.multimodal_config.mm_processor_cache_gb,
|
||||
@@ -142,8 +142,7 @@ def _compare_caches(
|
||||
for _ in range(int(item_capacity / hit_rate))
|
||||
]
|
||||
all_hashes = [
|
||||
MultiModalHasher.hash_kwargs(item=item.get_data())
|
||||
for item in all_items
|
||||
MultiModalHasher.hash_kwargs(item=item.get_data()) for item in all_items
|
||||
]
|
||||
|
||||
# Should not be used since there is nothing to convert to text
|
||||
@@ -162,7 +161,8 @@ def _compare_caches(
|
||||
for _ in range(is_cached_calls_per_iter):
|
||||
cache_0_p0.is_cached(selected_hashes)
|
||||
cache_0_p0_out = [
|
||||
item for item, _ in cache_0_p0.get_and_update(
|
||||
item
|
||||
for item, _ in cache_0_p0.get_and_update(
|
||||
[(item, prompt_update.content) for item in selected_items],
|
||||
selected_hashes,
|
||||
)
|
||||
@@ -174,7 +174,8 @@ def _compare_caches(
|
||||
for _ in range(is_cached_calls_per_iter):
|
||||
cache_1_p0.is_cached(selected_hashes)
|
||||
cache_1_p0_out = [
|
||||
item for item, _ in cache_1_p0.get_and_update(
|
||||
item
|
||||
for item, _ in cache_1_p0.get_and_update(
|
||||
[(item, prompt_update.content) for item in selected_items],
|
||||
selected_hashes,
|
||||
)
|
||||
@@ -183,14 +184,12 @@ def _compare_caches(
|
||||
if cache_0_p1 is None:
|
||||
cache_0_p1_out = cache_0_p0_out
|
||||
else:
|
||||
cache_0_p1_out = cache_0_p1.get_and_update(cache_0_p0_out,
|
||||
selected_hashes)
|
||||
cache_0_p1_out = cache_0_p1.get_and_update(cache_0_p0_out, selected_hashes)
|
||||
|
||||
if cache_1_p1 is None:
|
||||
cache_1_p1_out = cache_1_p0_out
|
||||
else:
|
||||
cache_1_p1_out = cache_1_p1.get_and_update(cache_1_p0_out,
|
||||
selected_hashes)
|
||||
cache_1_p1_out = cache_1_p1.get_and_update(cache_1_p0_out, selected_hashes)
|
||||
|
||||
assert cache_0_p1_out == cache_1_p1_out, f"Failed at {it=}"
|
||||
|
||||
|
||||
@@ -9,9 +9,6 @@ import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
from vllm.multimodal.processing import (
|
||||
InputProcessingContext,
|
||||
PlaceholderFeaturesInfo,
|
||||
@@ -24,8 +21,6 @@ from vllm.multimodal.processing import (
|
||||
iter_token_matches,
|
||||
replace_token_matches,
|
||||
)
|
||||
|
||||
# yapf: enable
|
||||
from vllm.multimodal.profiling import MultiModalProfiler
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
|
||||
@@ -34,7 +29,6 @@ from .utils import random_image
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("token_ids", "match_ids", "expected"),
|
||||
[
|
||||
@@ -44,34 +38,34 @@ pytestmark = pytest.mark.cpu_test
|
||||
[32000, 32000, 32000],
|
||||
[32000],
|
||||
[
|
||||
{ "start_idx": 0, "end_idx": 1 },
|
||||
{ "start_idx": 1, "end_idx": 2 },
|
||||
{ "start_idx": 2, "end_idx": 3 },
|
||||
{"start_idx": 0, "end_idx": 1},
|
||||
{"start_idx": 1, "end_idx": 2},
|
||||
{"start_idx": 2, "end_idx": 3},
|
||||
],
|
||||
),
|
||||
(
|
||||
[32000, 32000, 32000],
|
||||
[32000, 32000],
|
||||
[{ "start_idx": 0, "end_idx": 2 }],
|
||||
[{"start_idx": 0, "end_idx": 2}],
|
||||
),
|
||||
(
|
||||
[32000, 32000, 32000],
|
||||
[32000, 32000, 32000],
|
||||
[{ "start_idx": 0, "end_idx": 3 }],
|
||||
[{"start_idx": 0, "end_idx": 3}],
|
||||
),
|
||||
(
|
||||
[9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
|
||||
[28747, 32000],
|
||||
[
|
||||
{ "start_idx": 1, "end_idx": 3 },
|
||||
{ "start_idx": 6, "end_idx": 8 },
|
||||
{"start_idx": 1, "end_idx": 3},
|
||||
{"start_idx": 6, "end_idx": 8},
|
||||
],
|
||||
),
|
||||
(
|
||||
[9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
|
||||
[28747, 32000, 32000, 32000],
|
||||
[
|
||||
{ "start_idx": 1, "end_idx": 5 },
|
||||
{"start_idx": 1, "end_idx": 5},
|
||||
],
|
||||
),
|
||||
(
|
||||
@@ -82,14 +76,13 @@ pytestmark = pytest.mark.cpu_test
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("start_idx", [0, 4, 8])
|
||||
# yapf: enable
|
||||
def test_iter_token_matches(token_ids, match_ids, expected, start_idx):
|
||||
result = list(iter_token_matches(token_ids, match_ids,
|
||||
start_idx=start_idx))
|
||||
result = list(iter_token_matches(token_ids, match_ids, start_idx=start_idx))
|
||||
|
||||
# Manually constructed results
|
||||
assert [item._asdict() for item in result
|
||||
] == [item for item in expected if item["start_idx"] >= start_idx]
|
||||
assert [item._asdict() for item in result] == [
|
||||
item for item in expected if item["start_idx"] >= start_idx
|
||||
]
|
||||
|
||||
# Invariants
|
||||
match_lens = [end - start for start, end in result]
|
||||
@@ -97,7 +90,6 @@ def test_iter_token_matches(token_ids, match_ids, expected, start_idx):
|
||||
assert all(match_len == len(match_ids) for match_len in match_lens)
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("token_ids", "match_ids", "new_ids", "expected"),
|
||||
[
|
||||
@@ -141,7 +133,6 @@ def test_iter_token_matches(token_ids, match_ids, expected, start_idx):
|
||||
),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_replace_token_matches(token_ids, match_ids, new_ids, expected):
|
||||
result = replace_token_matches(token_ids, match_ids, new_ids)
|
||||
|
||||
@@ -149,7 +140,6 @@ def test_replace_token_matches(token_ids, match_ids, new_ids, expected):
|
||||
assert result == expected
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "target_by_key", "expected_by_key"),
|
||||
[
|
||||
@@ -166,11 +156,11 @@ def test_replace_token_matches(token_ids, match_ids, new_ids, expected):
|
||||
"pattern_1": [],
|
||||
"pattern_2": [],
|
||||
"pattern_3": [
|
||||
{ "start_idx": 0, "end_idx": 0 },
|
||||
{"start_idx": 0, "end_idx": 0},
|
||||
],
|
||||
"pattern_4": [],
|
||||
"pattern_5": [
|
||||
{ "start_idx": 0, "end_idx": 0 },
|
||||
{"start_idx": 0, "end_idx": 0},
|
||||
],
|
||||
},
|
||||
),
|
||||
@@ -186,26 +176,26 @@ def test_replace_token_matches(token_ids, match_ids, new_ids, expected):
|
||||
},
|
||||
{
|
||||
"pattern_1": [
|
||||
{ "start_idx": 0, "end_idx": 1 },
|
||||
{ "start_idx": 1, "end_idx": 2 },
|
||||
{ "start_idx": 2, "end_idx": 3 },
|
||||
{ "start_idx": 3, "end_idx": 4 },
|
||||
{"start_idx": 0, "end_idx": 1},
|
||||
{"start_idx": 1, "end_idx": 2},
|
||||
{"start_idx": 2, "end_idx": 3},
|
||||
{"start_idx": 3, "end_idx": 4},
|
||||
],
|
||||
"pattern_2": [
|
||||
{ "start_idx": 0, "end_idx": 2 },
|
||||
{ "start_idx": 2, "end_idx": 4 },
|
||||
{"start_idx": 0, "end_idx": 2},
|
||||
{"start_idx": 2, "end_idx": 4},
|
||||
],
|
||||
"pattern_3": [
|
||||
{ "start_idx": 0, "end_idx": 3 },
|
||||
{"start_idx": 0, "end_idx": 3},
|
||||
],
|
||||
"pattern_4": [
|
||||
{ "start_idx": 0, "end_idx": 0 },
|
||||
{"start_idx": 0, "end_idx": 0},
|
||||
],
|
||||
"pattern_5": [
|
||||
{ "start_idx": 1, "end_idx": 1 },
|
||||
{"start_idx": 1, "end_idx": 1},
|
||||
],
|
||||
"pattern_6": [
|
||||
{ "start_idx": 4, "end_idx": 4 },
|
||||
{"start_idx": 4, "end_idx": 4},
|
||||
],
|
||||
},
|
||||
),
|
||||
@@ -221,26 +211,25 @@ def test_replace_token_matches(token_ids, match_ids, new_ids, expected):
|
||||
},
|
||||
{
|
||||
"pattern_1": [
|
||||
{ "start_idx": 1, "end_idx": 3 },
|
||||
{ "start_idx": 6, "end_idx": 8 },
|
||||
{"start_idx": 1, "end_idx": 3},
|
||||
{"start_idx": 6, "end_idx": 8},
|
||||
],
|
||||
"pattern_2": [
|
||||
{ "start_idx": 1, "end_idx": 5 },
|
||||
{"start_idx": 1, "end_idx": 5},
|
||||
],
|
||||
"pattern_3": [],
|
||||
"pattern_4": [
|
||||
{ "start_idx": 0, "end_idx": 0 },
|
||||
{"start_idx": 0, "end_idx": 0},
|
||||
],
|
||||
"pattern_5": [],
|
||||
"pattern_6": [
|
||||
{ "start_idx": 10, "end_idx": 10 },
|
||||
{"start_idx": 10, "end_idx": 10},
|
||||
],
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
|
||||
# yapf: enable
|
||||
def test_find_token_matches(
|
||||
prompt,
|
||||
target_by_key,
|
||||
@@ -272,7 +261,6 @@ def test_find_token_matches(
|
||||
} == expected_by_key
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "target_by_key", "expected_by_key"),
|
||||
[
|
||||
@@ -288,16 +276,16 @@ def test_find_token_matches(
|
||||
"pattern_5": PromptIndexTargets.end(),
|
||||
},
|
||||
{
|
||||
"pattern_1": [{ "start_idx": 0, "end_idx": 0 }],
|
||||
"pattern_1": [{"start_idx": 0, "end_idx": 0}],
|
||||
"pattern_2": [],
|
||||
"pattern_3": [
|
||||
{ "start_idx": 0, "end_idx": 0 },
|
||||
{"start_idx": 0, "end_idx": 0},
|
||||
],
|
||||
"pattern_4": [],
|
||||
"pattern_5": [
|
||||
{ "start_idx": 0, "end_idx": 0 },
|
||||
{"start_idx": 0, "end_idx": 0},
|
||||
],
|
||||
}
|
||||
},
|
||||
),
|
||||
(
|
||||
"<image><image><image><image>",
|
||||
@@ -311,26 +299,26 @@ def test_find_token_matches(
|
||||
},
|
||||
{
|
||||
"pattern_1": [
|
||||
{ "start_idx": 0, "end_idx": 7 },
|
||||
{ "start_idx": 7, "end_idx": 14 },
|
||||
{ "start_idx": 14, "end_idx": 21 },
|
||||
{ "start_idx": 21, "end_idx": 28 },
|
||||
{"start_idx": 0, "end_idx": 7},
|
||||
{"start_idx": 7, "end_idx": 14},
|
||||
{"start_idx": 14, "end_idx": 21},
|
||||
{"start_idx": 21, "end_idx": 28},
|
||||
],
|
||||
"pattern_2": [
|
||||
{ "start_idx": 0, "end_idx": 14 },
|
||||
{ "start_idx": 14, "end_idx": 28 },
|
||||
{"start_idx": 0, "end_idx": 14},
|
||||
{"start_idx": 14, "end_idx": 28},
|
||||
],
|
||||
"pattern_3": [
|
||||
{ "start_idx": 0, "end_idx": 21 },
|
||||
{"start_idx": 0, "end_idx": 21},
|
||||
],
|
||||
"pattern_4": [
|
||||
{ "start_idx": 0, "end_idx": 0 },
|
||||
{"start_idx": 0, "end_idx": 0},
|
||||
],
|
||||
"pattern_5": [
|
||||
{ "start_idx": 7, "end_idx": 7 },
|
||||
{"start_idx": 7, "end_idx": 7},
|
||||
],
|
||||
"pattern_6": [
|
||||
{ "start_idx": 28, "end_idx": 28 },
|
||||
{"start_idx": 28, "end_idx": 28},
|
||||
],
|
||||
},
|
||||
),
|
||||
@@ -346,21 +334,21 @@ def test_find_token_matches(
|
||||
},
|
||||
{
|
||||
"pattern_1": [
|
||||
{ "start_idx": 0, "end_idx": 13 },
|
||||
{ "start_idx": 27, "end_idx": 40 },
|
||||
{"start_idx": 0, "end_idx": 13},
|
||||
{"start_idx": 27, "end_idx": 40},
|
||||
],
|
||||
"pattern_2": [
|
||||
{ "start_idx": 0, "end_idx": 27 },
|
||||
{"start_idx": 0, "end_idx": 27},
|
||||
],
|
||||
"pattern_3": [],
|
||||
"pattern_4": [
|
||||
{ "start_idx": 0, "end_idx": 0 },
|
||||
{"start_idx": 0, "end_idx": 0},
|
||||
],
|
||||
"pattern_5": [
|
||||
{ "start_idx": 13, "end_idx": 13 },
|
||||
{"start_idx": 13, "end_idx": 13},
|
||||
],
|
||||
"pattern_6": [
|
||||
{ "start_idx": 48, "end_idx": 48 },
|
||||
{"start_idx": 48, "end_idx": 48},
|
||||
],
|
||||
},
|
||||
),
|
||||
@@ -374,22 +362,21 @@ def test_find_token_matches(
|
||||
},
|
||||
{
|
||||
"pattern_1": [
|
||||
{ "start_idx": 0, "end_idx": 9 },
|
||||
{ "start_idx": 16, "end_idx": 25 },
|
||||
{"start_idx": 0, "end_idx": 9},
|
||||
{"start_idx": 16, "end_idx": 25},
|
||||
],
|
||||
"pattern_2": [
|
||||
{ "start_idx": 0, "end_idx": 16 },
|
||||
{ "start_idx": 16, "end_idx": 32 },
|
||||
{"start_idx": 0, "end_idx": 16},
|
||||
{"start_idx": 16, "end_idx": 32},
|
||||
],
|
||||
"pattern_3": [
|
||||
{ "start_idx": 0, "end_idx": 25 },
|
||||
{"start_idx": 0, "end_idx": 25},
|
||||
],
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
|
||||
# yapf: enable
|
||||
def test_find_text_matches(
|
||||
prompt,
|
||||
target_by_key,
|
||||
@@ -421,7 +408,6 @@ def test_find_text_matches(
|
||||
} == expected_by_key
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
|
||||
[
|
||||
@@ -549,9 +535,8 @@ def test_find_text_matches(
|
||||
},
|
||||
},
|
||||
),
|
||||
]
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_find_update_text(
|
||||
prompt,
|
||||
target_by_key,
|
||||
@@ -562,13 +547,15 @@ def test_find_update_text(
|
||||
mock_tokenizer = cast(AnyTokenizer, object())
|
||||
|
||||
for (
|
||||
update_type,
|
||||
expected_by_mm_count,
|
||||
update_type,
|
||||
expected_by_mm_count,
|
||||
) in expected_by_update_type_mm_count.items():
|
||||
for mm_count, expected in expected_by_mm_count.items():
|
||||
mm_prompt_updates = {
|
||||
key: [[update_type(key, target, repl_by_key[key]).resolve(i)]
|
||||
for i in range(mm_count)]
|
||||
key: [
|
||||
[update_type(key, target, repl_by_key[key]).resolve(i)]
|
||||
for i in range(mm_count)
|
||||
]
|
||||
for key, target in target_by_key.items()
|
||||
}
|
||||
|
||||
@@ -589,7 +576,6 @@ def test_find_update_text(
|
||||
assert new_prompt == expected
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
|
||||
[
|
||||
@@ -615,8 +601,43 @@ def test_find_update_text(
|
||||
{
|
||||
PromptInsertion: {
|
||||
0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
|
||||
1: [1, 9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918, 1550, 918, 1550], # noqa: E501
|
||||
2: [1, 9833, 28747, 32000, 32000, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918, 1550, 918, 1550, 1550, 918, 1550], # noqa: E501
|
||||
1: [
|
||||
1,
|
||||
9833,
|
||||
28747,
|
||||
32000,
|
||||
32000,
|
||||
32000,
|
||||
9833,
|
||||
28747,
|
||||
32000,
|
||||
32000,
|
||||
918,
|
||||
1550,
|
||||
918,
|
||||
1550,
|
||||
], # noqa: E501
|
||||
2: [
|
||||
1,
|
||||
9833,
|
||||
28747,
|
||||
32000,
|
||||
32000,
|
||||
32000,
|
||||
32000,
|
||||
32000,
|
||||
9833,
|
||||
28747,
|
||||
32000,
|
||||
32000,
|
||||
918,
|
||||
1550,
|
||||
918,
|
||||
1550,
|
||||
1550,
|
||||
918,
|
||||
1550,
|
||||
], # noqa: E501
|
||||
},
|
||||
PromptReplacement: {
|
||||
0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
|
||||
@@ -719,9 +740,8 @@ def test_find_update_text(
|
||||
},
|
||||
},
|
||||
),
|
||||
]
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_find_update_tokens(
|
||||
prompt,
|
||||
target_by_key,
|
||||
@@ -732,13 +752,15 @@ def test_find_update_tokens(
|
||||
mock_tokenizer = cast(AnyTokenizer, object())
|
||||
|
||||
for (
|
||||
update_type,
|
||||
expected_by_mm_count,
|
||||
update_type,
|
||||
expected_by_mm_count,
|
||||
) in expected_by_update_type_mm_count.items():
|
||||
for mm_count, expected in expected_by_mm_count.items():
|
||||
mm_prompt_updates = {
|
||||
key: [[update_type(key, target, repl_by_key[key]).resolve(i)]
|
||||
for i in range(mm_count)]
|
||||
key: [
|
||||
[update_type(key, target, repl_by_key[key]).resolve(i)]
|
||||
for i in range(mm_count)
|
||||
]
|
||||
for key, target in target_by_key.items()
|
||||
}
|
||||
|
||||
@@ -759,7 +781,6 @@ def test_find_update_tokens(
|
||||
assert new_prompt == expected
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
"repl_by_key",
|
||||
[
|
||||
@@ -796,8 +817,7 @@ def test_find_update_tokens(
|
||||
is_embed=None,
|
||||
),
|
||||
],
|
||||
}
|
||||
|
||||
},
|
||||
),
|
||||
(
|
||||
[1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],
|
||||
@@ -828,7 +848,7 @@ def test_find_update_tokens(
|
||||
),
|
||||
],
|
||||
# No match for pattern_4 as it has lower priority than pattern_1
|
||||
}
|
||||
},
|
||||
),
|
||||
(
|
||||
[1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
|
||||
@@ -867,12 +887,11 @@ def test_find_update_tokens(
|
||||
is_embed=None,
|
||||
),
|
||||
],
|
||||
}
|
||||
},
|
||||
),
|
||||
]
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
|
||||
# yapf: enable
|
||||
def test_find_mm_placeholders(
|
||||
repl_by_key,
|
||||
prompt,
|
||||
@@ -899,8 +918,15 @@ def test_find_mm_placeholders(
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
|
||||
@pytest.mark.parametrize(
|
||||
("limit", "num_supported", "is_valid"),
|
||||
[(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
|
||||
(2, 1, False), (2, 2, True)],
|
||||
[
|
||||
(0, 0, True),
|
||||
(0, 1, True),
|
||||
(1, 0, False),
|
||||
(1, 1, True),
|
||||
(1, 2, True),
|
||||
(2, 1, False),
|
||||
(2, 2, True),
|
||||
],
|
||||
)
|
||||
def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
|
||||
limit_mm_per_prompt = {"image": limit}
|
||||
@@ -930,8 +956,15 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
|
||||
@pytest.mark.parametrize(
|
||||
("num_images", "limit", "is_valid"),
|
||||
[(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
|
||||
(2, 1, False), (2, 2, True)],
|
||||
[
|
||||
(0, 0, True),
|
||||
(0, 1, True),
|
||||
(1, 0, False),
|
||||
(1, 1, True),
|
||||
(1, 2, True),
|
||||
(2, 1, False),
|
||||
(2, 2, True),
|
||||
],
|
||||
)
|
||||
def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
|
||||
limit_mm_per_prompt = {"image": limit}
|
||||
@@ -966,7 +999,6 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
|
||||
|
||||
|
||||
class DummyProcessor:
|
||||
|
||||
def __init__(self, a: int = 0, b: int = 0) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -982,7 +1014,6 @@ class DummyProcessor:
|
||||
return dict(a=a, c=c)
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"]) # Dummy
|
||||
@pytest.mark.parametrize(
|
||||
("config_kwargs", "inference_kwargs", "expected_kwargs"),
|
||||
@@ -996,7 +1027,6 @@ class DummyProcessor:
|
||||
({"b": 1, "c": 1}, {}, {"a": 0, "b": 1}),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_hf_processor_init_kwargs(
|
||||
model_id,
|
||||
config_kwargs,
|
||||
@@ -1020,7 +1050,6 @@ def test_hf_processor_init_kwargs(
|
||||
assert getattr(processor, k) == v
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"]) # Dummy
|
||||
@pytest.mark.parametrize(
|
||||
("config_kwargs", "inference_kwargs", "expected_kwargs"),
|
||||
@@ -1034,7 +1063,6 @@ def test_hf_processor_init_kwargs(
|
||||
({"b": 1, "c": 1}, {}, {"a": 0, "c": 1}),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_hf_processor_call_kwargs(
|
||||
model_id,
|
||||
config_kwargs,
|
||||
|
||||
@@ -233,7 +233,6 @@ async def test_fetch_video_http_with_dynamic_loader(
|
||||
assert metadata_sync["video_backend"] == "opencv_dynamic"
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
@@ -264,7 +263,6 @@ async def test_fetch_video_http_with_dynamic_loader(
|
||||
("image", 0),
|
||||
],
|
||||
),
|
||||
|
||||
# Two modalities
|
||||
## Internally sorted
|
||||
dict(
|
||||
@@ -276,7 +274,7 @@ async def test_fetch_video_http_with_dynamic_loader(
|
||||
"audio": [
|
||||
PlaceholderRange(offset=0, length=2),
|
||||
PlaceholderRange(offset=2, length=3),
|
||||
]
|
||||
],
|
||||
},
|
||||
expected_modality_idxs=[
|
||||
("audio", 0),
|
||||
@@ -295,7 +293,7 @@ async def test_fetch_video_http_with_dynamic_loader(
|
||||
"audio": [
|
||||
PlaceholderRange(offset=5, length=2),
|
||||
PlaceholderRange(offset=11, length=4),
|
||||
]
|
||||
],
|
||||
},
|
||||
expected_modality_idxs=[
|
||||
("image", 0),
|
||||
@@ -314,7 +312,7 @@ async def test_fetch_video_http_with_dynamic_loader(
|
||||
"audio": [
|
||||
PlaceholderRange(offset=11, length=4),
|
||||
PlaceholderRange(offset=5, length=2),
|
||||
]
|
||||
],
|
||||
},
|
||||
expected_modality_idxs=[
|
||||
("image", 1),
|
||||
@@ -323,7 +321,6 @@ async def test_fetch_video_http_with_dynamic_loader(
|
||||
("audio", 0),
|
||||
],
|
||||
),
|
||||
|
||||
# Three modalities
|
||||
## Internally sorted
|
||||
dict(
|
||||
@@ -339,7 +336,7 @@ async def test_fetch_video_http_with_dynamic_loader(
|
||||
PlaceholderRange(offset=3, length=4),
|
||||
PlaceholderRange(offset=7, length=5),
|
||||
PlaceholderRange(offset=12, length=6),
|
||||
]
|
||||
],
|
||||
},
|
||||
expected_modality_idxs=[
|
||||
("audio", 0),
|
||||
@@ -363,7 +360,7 @@ async def test_fetch_video_http_with_dynamic_loader(
|
||||
],
|
||||
"video": [
|
||||
PlaceholderRange(offset=8, length=5),
|
||||
]
|
||||
],
|
||||
},
|
||||
expected_modality_idxs=[
|
||||
("image", 0),
|
||||
@@ -386,7 +383,7 @@ async def test_fetch_video_http_with_dynamic_loader(
|
||||
],
|
||||
"video": [
|
||||
PlaceholderRange(offset=8, length=5),
|
||||
]
|
||||
],
|
||||
},
|
||||
expected_modality_idxs=[
|
||||
("image", 0),
|
||||
@@ -398,7 +395,6 @@ async def test_fetch_video_http_with_dynamic_loader(
|
||||
),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_argsort_mm_positions(case):
|
||||
mm_positions = case["mm_positions"]
|
||||
expected_modality_idxs = case["expected_modality_idxs"]
|
||||
@@ -413,13 +409,16 @@ def test_argsort_mm_positions(case):
|
||||
@pytest.mark.parametrize("num_frames", [-1, 32, 1800])
|
||||
async def test_allowed_media_domains(video_url: str, num_frames: int):
|
||||
connector = MediaConnector(
|
||||
media_io_kwargs={"video": {
|
||||
"num_frames": num_frames,
|
||||
}},
|
||||
media_io_kwargs={
|
||||
"video": {
|
||||
"num_frames": num_frames,
|
||||
}
|
||||
},
|
||||
allowed_media_domains=[
|
||||
"www.bogotobogo.com",
|
||||
"github.com",
|
||||
])
|
||||
],
|
||||
)
|
||||
|
||||
video_sync, metadata_sync = connector.fetch_video(video_url)
|
||||
video_async, metadata_async = await connector.fetch_video_async(video_url)
|
||||
|
||||
+50
-42
@@ -59,48 +59,52 @@ def test_parse_raw_single_batch_string_slice(inputs_slice: slice):
|
||||
)
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize('mm_processor_kwargs,expected_mm_kwargs', [
|
||||
(None, [{}, {}]),
|
||||
({}, [{}, {}]),
|
||||
({"foo": 100}, [{"foo": 100}, {"foo": 100}]),
|
||||
([{"foo": 100}, {"bar": 200}], [{"foo": 100}, {"bar": 200}]),
|
||||
])
|
||||
# yapf: enable
|
||||
@pytest.mark.parametrize(
|
||||
"mm_processor_kwargs,expected_mm_kwargs",
|
||||
[
|
||||
(None, [{}, {}]),
|
||||
({}, [{}, {}]),
|
||||
({"foo": 100}, [{"foo": 100}, {"foo": 100}]),
|
||||
([{"foo": 100}, {"bar": 200}], [{"foo": 100}, {"bar": 200}]),
|
||||
],
|
||||
)
|
||||
def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):
|
||||
"""Test mm_processor_kwargs init for zipping enc/dec prompts."""
|
||||
encoder_prompts = ['An encoder prompt', 'Another encoder prompt']
|
||||
decoder_prompts = ['A decoder prompt', 'Another decoder prompt']
|
||||
zipped_prompts = zip_enc_dec_prompts(encoder_prompts, decoder_prompts,
|
||||
mm_processor_kwargs)
|
||||
encoder_prompts = ["An encoder prompt", "Another encoder prompt"]
|
||||
decoder_prompts = ["A decoder prompt", "Another decoder prompt"]
|
||||
zipped_prompts = zip_enc_dec_prompts(
|
||||
encoder_prompts, decoder_prompts, mm_processor_kwargs
|
||||
)
|
||||
assert len(zipped_prompts) == len(encoder_prompts) == len(decoder_prompts)
|
||||
for enc, dec, exp_kwargs, zipped in zip(encoder_prompts, decoder_prompts,
|
||||
expected_mm_kwargs,
|
||||
zipped_prompts):
|
||||
for enc, dec, exp_kwargs, zipped in zip(
|
||||
encoder_prompts, decoder_prompts, expected_mm_kwargs, zipped_prompts
|
||||
):
|
||||
assert isinstance(zipped, dict)
|
||||
assert len(zipped.keys()) == 3
|
||||
assert zipped['encoder_prompt'] == enc
|
||||
assert zipped['decoder_prompt'] == dec
|
||||
assert zipped['mm_processor_kwargs'] == exp_kwargs
|
||||
assert zipped["encoder_prompt"] == enc
|
||||
assert zipped["decoder_prompt"] == dec
|
||||
assert zipped["mm_processor_kwargs"] == exp_kwargs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [
|
||||
"facebook/opt-125m",
|
||||
])
|
||||
@pytest.mark.parametrize("prompt", [
|
||||
{
|
||||
"prompt": "",
|
||||
"multi_modal_data": {
|
||||
"dummy": []
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
"facebook/opt-125m",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"prompt",
|
||||
[
|
||||
{
|
||||
"prompt": "",
|
||||
"multi_modal_data": {"dummy": []},
|
||||
},
|
||||
},
|
||||
{
|
||||
"prompt_token_ids": [],
|
||||
"multi_modal_data": {
|
||||
"dummy": []
|
||||
{
|
||||
"prompt_token_ids": [],
|
||||
"multi_modal_data": {"dummy": []},
|
||||
},
|
||||
},
|
||||
])
|
||||
],
|
||||
)
|
||||
def test_preprocessor_text_no_mm_inputs(model_id, prompt):
|
||||
model_config = ModelConfig(model=model_id)
|
||||
tokenizer = init_tokenizer_from_configs(model_config)
|
||||
@@ -110,15 +114,19 @@ def test_preprocessor_text_no_mm_inputs(model_id, prompt):
|
||||
input_preprocessor.preprocess(prompt)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [
|
||||
"facebook/chameleon-7b",
|
||||
])
|
||||
@pytest.mark.parametrize("prompt", [
|
||||
"",
|
||||
{
|
||||
"prompt_token_ids": []
|
||||
},
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
"facebook/chameleon-7b",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"prompt",
|
||||
[
|
||||
"",
|
||||
{"prompt_token_ids": []},
|
||||
],
|
||||
)
|
||||
def test_preprocessor_always_mm_code_path(model_id, prompt):
|
||||
model_config = ModelConfig(model=model_id)
|
||||
tokenizer = init_tokenizer_from_configs(model_config)
|
||||
|
||||
@@ -9,14 +9,10 @@ import pytest
|
||||
import torch
|
||||
import torch_xla
|
||||
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
from vllm.model_executor.layers.fused_moe.moe_pallas import fused_moe as pallas_moe
|
||||
from vllm.model_executor.layers.fused_moe.moe_torch_iterative import (
|
||||
fused_moe as torch_moe,
|
||||
)
|
||||
|
||||
# yapf: enable
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_tpu():
|
||||
|
||||
+70
-61
@@ -388,7 +388,6 @@ def test_duplicate_dict_args(caplog_vllm, parser):
|
||||
assert "-O.level" in caplog_vllm.text
|
||||
|
||||
|
||||
# yapf: enable
|
||||
@pytest.mark.parametrize(
|
||||
"callable,kw_name,requires_kw_only,allow_var_kwargs,is_supported",
|
||||
[
|
||||
@@ -408,7 +407,6 @@ def test_duplicate_dict_args(caplog_vllm, parser):
|
||||
(lambda foo, **kwargs: None, "foo", True, True, False),
|
||||
],
|
||||
)
|
||||
# yapf: disable
|
||||
def test_supports_kw(
|
||||
callable, kw_name, requires_kw_only, allow_var_kwargs, is_supported
|
||||
):
|
||||
@@ -681,7 +679,6 @@ def test_lru_cache():
|
||||
assert 6 in cache
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("src_dtype", "tgt_dtype", "expected_result"),
|
||||
[
|
||||
@@ -715,12 +712,10 @@ def test_lru_cache():
|
||||
(torch.complex64, torch.complex32, False),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_is_lossless_cast(src_dtype, tgt_dtype, expected_result):
|
||||
assert is_lossless_cast(src_dtype, tgt_dtype) == expected_result
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("dtypes", "expected_result"),
|
||||
[
|
||||
@@ -730,7 +725,6 @@ def test_is_lossless_cast(src_dtype, tgt_dtype, expected_result):
|
||||
([torch.bool, torch.int8, torch.float16, torch.complex32], torch.complex32), # noqa: E501
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_common_broadcastable_dtype(dtypes, expected_result):
|
||||
assert common_broadcastable_dtype(dtypes) == expected_result
|
||||
|
||||
@@ -775,7 +769,6 @@ def test_placeholder_module_error_handling():
|
||||
_ = placeholder_attr.module
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
"obj,key1,key2",
|
||||
[
|
||||
@@ -785,8 +778,8 @@ def test_placeholder_module_error_handling():
|
||||
({1: "a", 2: "b"}, 1, 3),
|
||||
# Tests for both keys do not exist
|
||||
({1: "a", 2: "b"}, 3, 4),
|
||||
])
|
||||
# yapf: enable
|
||||
],
|
||||
)
|
||||
def test_swap_dict_values(obj, key1, key2):
|
||||
original_obj = obj.copy()
|
||||
swap_dict_values(obj, key1, key2)
|
||||
@@ -800,26 +793,30 @@ def test_swap_dict_values(obj, key1, key2):
|
||||
assert key1 not in obj
|
||||
|
||||
|
||||
def test_model_specification(parser_with_config, cli_config_file,
|
||||
cli_config_file_with_model):
|
||||
def test_model_specification(
|
||||
parser_with_config, cli_config_file, cli_config_file_with_model
|
||||
):
|
||||
# Test model in CLI takes precedence over config
|
||||
args = parser_with_config.parse_args(
|
||||
['serve', 'cli-model', '--config', cli_config_file_with_model])
|
||||
assert args.model_tag == 'cli-model'
|
||||
assert args.served_model_name == 'mymodel'
|
||||
["serve", "cli-model", "--config", cli_config_file_with_model]
|
||||
)
|
||||
assert args.model_tag == "cli-model"
|
||||
assert args.served_model_name == "mymodel"
|
||||
|
||||
# Test model from config file works
|
||||
args = parser_with_config.parse_args([
|
||||
'serve',
|
||||
'--config',
|
||||
cli_config_file_with_model,
|
||||
])
|
||||
assert args.model == 'config-model'
|
||||
assert args.served_model_name == 'mymodel'
|
||||
args = parser_with_config.parse_args(
|
||||
[
|
||||
"serve",
|
||||
"--config",
|
||||
cli_config_file_with_model,
|
||||
]
|
||||
)
|
||||
assert args.model == "config-model"
|
||||
assert args.served_model_name == "mymodel"
|
||||
|
||||
# Test no model specified anywhere raises error
|
||||
with pytest.raises(ValueError, match="No model specified!"):
|
||||
parser_with_config.parse_args(['serve', '--config', cli_config_file])
|
||||
parser_with_config.parse_args(["serve", "--config", cli_config_file])
|
||||
|
||||
# Test using --model option raises error
|
||||
# with pytest.raises(
|
||||
@@ -833,47 +830,52 @@ def test_model_specification(parser_with_config, cli_config_file,
|
||||
# Test using --model option back-compatibility
|
||||
# (when back-compatibility ends, the above test should be uncommented
|
||||
# and the below test should be removed)
|
||||
args = parser_with_config.parse_args([
|
||||
'serve',
|
||||
'--tensor-parallel-size',
|
||||
'2',
|
||||
'--model',
|
||||
'my-model',
|
||||
'--trust-remote-code',
|
||||
'--port',
|
||||
'8001',
|
||||
])
|
||||
args = parser_with_config.parse_args(
|
||||
[
|
||||
"serve",
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--model",
|
||||
"my-model",
|
||||
"--trust-remote-code",
|
||||
"--port",
|
||||
"8001",
|
||||
]
|
||||
)
|
||||
assert args.model is None
|
||||
assert args.tensor_parallel_size == 2
|
||||
assert args.trust_remote_code is True
|
||||
assert args.port == 8001
|
||||
|
||||
args = parser_with_config.parse_args([
|
||||
'serve',
|
||||
'--tensor-parallel-size=2',
|
||||
'--model=my-model',
|
||||
'--trust-remote-code',
|
||||
'--port=8001',
|
||||
])
|
||||
args = parser_with_config.parse_args(
|
||||
[
|
||||
"serve",
|
||||
"--tensor-parallel-size=2",
|
||||
"--model=my-model",
|
||||
"--trust-remote-code",
|
||||
"--port=8001",
|
||||
]
|
||||
)
|
||||
assert args.model is None
|
||||
assert args.tensor_parallel_size == 2
|
||||
assert args.trust_remote_code is True
|
||||
assert args.port == 8001
|
||||
|
||||
# Test other config values are preserved
|
||||
args = parser_with_config.parse_args([
|
||||
'serve',
|
||||
'cli-model',
|
||||
'--config',
|
||||
cli_config_file_with_model,
|
||||
])
|
||||
args = parser_with_config.parse_args(
|
||||
[
|
||||
"serve",
|
||||
"cli-model",
|
||||
"--config",
|
||||
cli_config_file_with_model,
|
||||
]
|
||||
)
|
||||
assert args.tensor_parallel_size == 2
|
||||
assert args.trust_remote_code is True
|
||||
assert args.port == 12312
|
||||
|
||||
|
||||
@pytest.mark.parametrize("input", [(), ("abc", ), (None, ),
|
||||
(None, bool, [1, 2, 3])])
|
||||
@pytest.mark.parametrize("input", [(), ("abc",), (None,), (None, bool, [1, 2, 3])])
|
||||
def test_sha256(input: tuple):
|
||||
digest = sha256(input)
|
||||
assert digest is not None
|
||||
@@ -887,7 +889,7 @@ def test_sha256(input: tuple):
|
||||
assert digest == sha256(input)
|
||||
|
||||
# hashing different input, returns different value
|
||||
assert digest != sha256(input + (1, ))
|
||||
assert digest != sha256(input + (1,))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -897,7 +899,8 @@ def test_sha256(input: tuple):
|
||||
("tcp://127.0.0.1:5555", ("tcp", "127.0.0.1", "5555")),
|
||||
("tcp://[::1]:5555", ("tcp", "::1", "5555")), # IPv6 address
|
||||
("inproc://some_identifier", ("inproc", "some_identifier", "")),
|
||||
])
|
||||
],
|
||||
)
|
||||
def test_split_zmq_path(path, expected):
|
||||
assert split_zmq_path(path) == expected
|
||||
|
||||
@@ -909,7 +912,8 @@ def test_split_zmq_path(path, expected):
|
||||
"tcp://127.0.0.1", # Missing port
|
||||
"tcp://[::1]", # Missing port for IPv6
|
||||
"tcp://:5555", # Missing host
|
||||
])
|
||||
],
|
||||
)
|
||||
def test_split_zmq_path_invalid(invalid_path):
|
||||
with pytest.raises(ValueError):
|
||||
split_zmq_path(invalid_path)
|
||||
@@ -931,8 +935,9 @@ def test_make_zmq_socket_ipv6():
|
||||
zsock: zmq.Socket = make_zmq_socket(ctx, ipv6_path, socket_type)
|
||||
|
||||
# Verify that the IPV6 option is set
|
||||
assert zsock.getsockopt(
|
||||
zmq.IPV6) == 1, "IPV6 option should be enabled for IPv6 addresses"
|
||||
assert zsock.getsockopt(zmq.IPV6) == 1, (
|
||||
"IPV6 option should be enabled for IPv6 addresses"
|
||||
)
|
||||
|
||||
# Clean up
|
||||
zsock.close()
|
||||
@@ -1019,15 +1024,14 @@ def test_convert_ids_list_to_tokens():
|
||||
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-1.5B-Instruct")
|
||||
token_ids = tokenizer.encode("Hello, world!")
|
||||
# token_ids = [9707, 11, 1879, 0]
|
||||
assert tokenizer.convert_ids_to_tokens(token_ids) == [
|
||||
'Hello', ',', 'Ġworld', '!'
|
||||
]
|
||||
assert tokenizer.convert_ids_to_tokens(token_ids) == ["Hello", ",", "Ġworld", "!"]
|
||||
tokens = convert_ids_list_to_tokens(tokenizer, token_ids)
|
||||
assert tokens == ['Hello', ',', ' world', '!']
|
||||
assert tokens == ["Hello", ",", " world", "!"]
|
||||
|
||||
|
||||
def test_current_stream_multithread():
|
||||
import threading
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
|
||||
@@ -1046,13 +1050,18 @@ def test_current_stream_multithread():
|
||||
child_thread.start()
|
||||
|
||||
try:
|
||||
assert thread_stream_ready.wait(
|
||||
timeout=5), "Child thread failed to enter stream context in time"
|
||||
assert thread_stream_ready.wait(timeout=5), (
|
||||
"Child thread failed to enter stream context in time"
|
||||
)
|
||||
|
||||
main_current_stream = current_stream()
|
||||
|
||||
assert main_current_stream != child_stream, "Main thread's current_stream was contaminated by child thread"
|
||||
assert main_current_stream == main_default_stream, "Main thread's current_stream is not the default stream"
|
||||
assert main_current_stream != child_stream, (
|
||||
"Main thread's current_stream was contaminated by child thread"
|
||||
)
|
||||
assert main_current_stream == main_default_stream, (
|
||||
"Main thread's current_stream is not the default stream"
|
||||
)
|
||||
|
||||
# Notify child thread it can exit
|
||||
thread_can_exit.set()
|
||||
@@ -1070,7 +1079,7 @@ def test_load_config_file(tmp_path):
|
||||
"enable-logging": True,
|
||||
"list-arg": ["item1", "item2"],
|
||||
"port": 12323,
|
||||
"tensor-parallel-size": 4
|
||||
"tensor-parallel-size": 4,
|
||||
}
|
||||
|
||||
# Write the configuration data to a temporary YAML file
|
||||
|
||||
@@ -16,9 +16,6 @@ from vllm.multimodal.inputs import (
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import GiB_bytes, sha256, sha256_cbor
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
|
||||
# disable yapf here as it formats differently than isort such that both fail
|
||||
# yapf: disable
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
FreeKVCacheBlockQueue,
|
||||
@@ -48,8 +45,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
from vllm.v1.metrics.stats import PrefixCacheStats
|
||||
from vllm.v1.request import Request
|
||||
|
||||
# yapf: enable
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
|
||||
@@ -22,8 +22,6 @@ from vllm.config import VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import is_pin_memory_available
|
||||
|
||||
# yapf: disable
|
||||
from vllm.v1.sample.logits_processor import (
|
||||
BatchUpdate,
|
||||
BatchUpdateBuilder,
|
||||
@@ -34,8 +32,6 @@ from vllm.v1.sample.logits_processor import (
|
||||
MoveDirectionality,
|
||||
build_logitsprocs,
|
||||
)
|
||||
|
||||
# yapf: enable
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
|
||||
PIN_MEMORY_AVAILABLE = is_pin_memory_available()
|
||||
|
||||
@@ -7,8 +7,6 @@ from typing import Union
|
||||
import pytest
|
||||
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
|
||||
# yapf: disable
|
||||
from tests.v1.logits_processors.utils import (
|
||||
DUMMY_LOGITPROC_ARG,
|
||||
DUMMY_LOGITPROC_FQCN,
|
||||
@@ -24,8 +22,6 @@ from tests.v1.logits_processors.utils import (
|
||||
prompts,
|
||||
)
|
||||
from tests.v1.logits_processors.utils import entry_points as fake_entry_points
|
||||
|
||||
# yapf: enable
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.v1.sample.logits_processor import (
|
||||
STR_POOLING_REJECTS_LOGITSPROCS,
|
||||
|
||||
@@ -11,8 +11,6 @@ import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServerCustom, create_new_process_for_each_test
|
||||
|
||||
# yapf: disable
|
||||
from tests.v1.logits_processors.utils import (
|
||||
DUMMY_LOGITPROC_ARG,
|
||||
DUMMY_LOGITPROC_FQCN,
|
||||
@@ -25,8 +23,6 @@ from tests.v1.logits_processors.utils import (
|
||||
)
|
||||
from tests.v1.logits_processors.utils import entry_points as fake_entry_points
|
||||
|
||||
# yapf: enable
|
||||
|
||||
|
||||
def _server_with_logitproc_entrypoint(
|
||||
env_dict: Optional[dict[str, str]],
|
||||
|
||||
Reference in New Issue
Block a user