[CI] Consolidate multimodal entrypoint tests. (#44819)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-06-08 04:48:08 -07:00
committed by GitHub
parent 980796cd07
commit 93ee4cd47f
24 changed files with 107 additions and 62 deletions
-42
View File
@@ -4,7 +4,6 @@ import weakref
import pytest
from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.sampling_params import SamplingParams
@@ -76,47 +75,6 @@ def test_multi_chat(text_llm):
assert len(outputs) == 2
@pytest.fixture(scope="function")
def vision_llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model="microsoft/Phi-3.5-vision-instruct",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
trust_remote_code=True,
limit_mm_per_prompt={"image": 2},
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
@pytest.mark.parametrize(
"image_urls", [[TEST_IMAGE_ASSETS[0], TEST_IMAGE_ASSETS[1]]], indirect=True
)
def test_chat_multi_image(vision_llm, image_urls: list[str]):
messages = [
{
"role": "user",
"content": [
*(
{"type": "image_url", "image_url": {"url": image_url}}
for image_url in image_urls
),
{"type": "text", "text": "What's in this image?"},
],
}
]
outputs = vision_llm.chat(messages)
assert len(outputs) >= 0
def test_llm_chat_tokenization_no_double_bos(text_llm):
"""
LLM.chat() should not add special tokens when using chat templates.
@@ -1,193 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test that ``InputProcessor.inject_into_mm_cache()`` correctly injects
pre-processed mm_kwargs into the processor cache and reports MM cache
hit rate metrics accurately.
This is used by frameworks like Dynamo that run the HF processor on a
frontend and transfer pre-processed mm_kwargs to the backend, avoiding
redundant processing.
"""
import logging
import pytest
import regex as re
from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
from vllm import LLM, SamplingParams
from vllm.renderers.params import ChatParams
from vllm.v1.metrics import loggers as stat_loggers
from vllm.v1.metrics.reader import Counter, Metric
def _make_messages(image_url: str):
return [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {"url": image_url},
},
],
}
]
def _get_counter_value(metrics: list[Metric], name: str):
metric = next(m for m in metrics if m.name == name)
assert isinstance(metric, Counter)
return metric.value
def _get_mm_cache_stats(metrics: list[Metric]):
mm_cache_queries = _get_counter_value(metrics, "vllm:mm_cache_queries")
mm_cache_hits = _get_counter_value(metrics, "vllm:mm_cache_hits")
return mm_cache_queries, mm_cache_hits
def _get_mm_cache_log(llm: LLM, caplog_vllm: pytest.LogCaptureFixture) -> float:
caplog_vllm.clear()
with caplog_vllm.at_level(logging.INFO, logger=stat_loggers.__name__):
llm.llm_engine.do_log_stats()
assert len(caplog_vllm.records) == 1
msg = caplog_vllm.records[0].getMessage()
assert "MM cache hit rate" in msg
match = re.search(r"MM cache hit rate: ([0-9.]+)%", msg)
assert match is not None
return float(match.group(1))
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:2]], indirect=True)
@pytest.mark.parametrize("mm_processor_cache_type", ["lru", "shm"])
def test_inject_into_mm_cache(
num_gpus_available,
image_urls,
mm_processor_cache_type,
caplog_vllm,
):
"""Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
the processor cache and MM cache hit metrics are updated correctly.
Steps:
1. Two normal requests (same image) -> cache miss then hit (baseline)
2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
then generate with a pre-rendered input -> verifies injection works
"""
llm = LLM(
model="llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
mm_processor_cache_type=mm_processor_cache_type,
)
# Step 1: Normal requests to populate the cache
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(50.0)
# Step 2: Use a second image to get valid expanded tokens and
# placeholder positions via the renderer.
llm.chat(_make_messages(image_urls[1]))
queries_before = _get_mm_cache_stats(llm.get_metrics())[0] # 3
renderer = llm.llm_engine.renderer
cache = renderer.mm_processor_cache
assert cache is not None, "Processor cache should be enabled"
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[1])],
ChatParams(),
)
eng_input = eng_prompts[0]
# Inject pre-processed mm_kwargs with a NEW hash via public API
new_mm_hash = "deadbeef" * 8
mm_hashes = {"image": [new_mm_hash]}
mm_kwargs = eng_input["mm_kwargs"]
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# Build pre-rendered input (no externally_processed flag needed)
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
# Verify cache was queried and injection happened
queries_after = _get_mm_cache_stats(llm.get_metrics())[0]
assert queries_after > queries_before, (
"Cache should have been queried for the injected item"
)
mm_rate = _get_mm_cache_log(llm, caplog_vllm)
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True)
def test_inject_into_mm_cache_without_cache(
num_gpus_available,
image_urls,
):
"""Test that inject_into_mm_cache works gracefully when processor cache
is disabled (mm_processor_cache_gb=0). Should not crash.
"""
llm = LLM(
model="llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
mm_processor_cache_gb=0,
)
# Run a normal chat request first to warm up the model.
llm.chat(_make_messages(image_urls[0]))
# Use the renderer to get a proper EngineInput with expanded tokens
renderer = llm.llm_engine.renderer
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[0])],
ChatParams(),
)
eng_input = eng_prompts[0]
mm_hashes = {"image": ["abcd1234" * 8]}
mm_kwargs = eng_input["mm_kwargs"]
# inject_into_mm_cache should not crash even without cache
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# Build and generate with pre-rendered input
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
result = llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
assert len(result) == 1, "Should produce one output"
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
@@ -1,97 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import pytest
import regex as re
from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
from vllm import LLM
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.v1.metrics import loggers as stat_loggers
from vllm.v1.metrics.reader import Counter, Metric
def _make_messages(image_url: str) -> list[ChatCompletionMessageParam]:
return [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {"url": image_url},
},
],
}
]
def _get_counter_value(metrics: list[Metric], name: str):
metric = next(m for m in metrics if m.name == name)
assert isinstance(metric, Counter)
return metric.value
def _get_mm_cache_stats(metrics: list[Metric]):
mm_cache_queries = _get_counter_value(metrics, "vllm:mm_cache_queries")
mm_cache_hits = _get_counter_value(metrics, "vllm:mm_cache_hits")
return mm_cache_queries, mm_cache_hits
def _get_mm_cache_log(llm: LLM, caplog_vllm: pytest.LogCaptureFixture) -> float:
caplog_vllm.clear()
with caplog_vllm.at_level(logging.INFO, logger=stat_loggers.__name__):
llm.llm_engine.do_log_stats()
assert len(caplog_vllm.records) == 1
msg = caplog_vllm.records[0].getMessage()
assert "MM cache hit rate" in msg
match = re.search(r"MM cache hit rate: ([0-9.]+)%", msg)
assert match is not None
return float(match.group(1))
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:2]], indirect=True)
@pytest.mark.parametrize("mm_processor_cache_type", ["lru", "shm"])
def test_mm_cache_stats(
num_gpus_available,
image_urls,
mm_processor_cache_type,
caplog_vllm,
):
llm = LLM(
model="llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
mm_processor_cache_type=mm_processor_cache_type,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 0)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (3, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(33.3)
# NOTE: This only resets hit rate stats in CachingMetrics
# The raw queries and hits counts remain unaffected
llm.reset_mm_cache()
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (4, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(llm.get_metrics()) == (5, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
from vllm.distributed import cleanup_dist_env_and_memory
MODEL = "llava-hf/llava-1.5-7b-hf"
PROMPT = "USER: <image>\nDescribe this image briefly.\nASSISTANT:"
TEXT_ONLY_PROMPT = "USER: What is 2 + 2?\nASSISTANT:"
@pytest.fixture(scope="module")
def llm():
"""LLM with enable_mm_embeds=True and all modality limits zeroed out."""
llm = LLM(
model=MODEL,
max_model_len=2048,
enforce_eager=True,
gpu_memory_utilization=0.8,
enable_mm_embeds=True,
limit_mm_per_prompt={"image": 0},
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
@pytest.mark.skip_global_cleanup
def test_generate_with_embedding(llm: LLM):
"""Pre-computed embedding produces tokens without hanging."""
embedding = ImageAsset("stop_sign").image_embeds
outputs = llm.generate(
{"prompt": PROMPT, "multi_modal_data": {"image": embedding}},
sampling_params=SamplingParams(max_tokens=32, temperature=0.0),
)
assert len(outputs) == 1
assert len(outputs[0].outputs[0].text) > 0
@pytest.mark.skip_global_cleanup
def test_raw_image_rejected(llm: LLM):
"""Raw image input is still rejected when limit=0."""
raw_image = ImageAsset("stop_sign").pil_image
with pytest.raises(ValueError, match=r"At most 0 image\(s\)"):
llm.generate(
{"prompt": PROMPT, "multi_modal_data": {"image": raw_image}},
sampling_params=SamplingParams(max_tokens=16),
)
@pytest.mark.skip_global_cleanup
def test_text_only_prompt(llm: LLM):
"""Text-only prompts still work under this config."""
outputs = llm.generate(
TEXT_ONLY_PROMPT,
sampling_params=SamplingParams(max_tokens=16, temperature=0.0),
)
assert len(outputs) == 1
assert len(outputs[0].outputs[0].text) > 0
@@ -1,288 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from vllm import LLM, SamplingParams
def _make_mock_llm() -> LLM:
llm = object.__new__(LLM)
llm.model_config = SimpleNamespace(
runner_type="generate", enable_prompt_embeds=False
)
return llm
def test_generate_forwards_mm_processor_kwargs() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 4}
sampling_params = SamplingParams(max_tokens=1)
llm._run_completion = Mock(return_value=["ok"])
outputs = llm.generate(
"prompt",
sampling_params=sampling_params,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == ["ok"]
assert llm._run_completion.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_enqueue_forwards_mm_processor_kwargs() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_resize": False}
sampling_params = SamplingParams(max_tokens=1)
llm._add_completion_requests = Mock(return_value=["req-0"])
request_ids = llm.enqueue(
"prompt",
sampling_params=sampling_params,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert request_ids == ["req-0"]
assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_chat_forwards_mm_processor_kwargs() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_pan_and_scan": True}
sampling_params = SamplingParams(max_tokens=1)
messages = [{"role": "user", "content": "hello"}]
llm._run_chat = Mock(return_value=["ok"])
outputs = llm.chat(
messages,
sampling_params=sampling_params,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == ["ok"]
assert llm._run_chat.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_enqueue_chat_forwards_mm_processor_kwargs() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_pan_and_scan": True}
sampling_params = SamplingParams(max_tokens=1)
messages = [{"role": "user", "content": "hello"}]
llm._add_chat_requests = Mock(return_value=["req-0"])
request_ids = llm.enqueue_chat(
messages,
sampling_params=sampling_params,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert request_ids == ["req-0"]
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_run_chat_forwards_mm_processor_kwargs() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
sampling_params = SamplingParams(max_tokens=1)
messages = [{"role": "user", "content": "hello"}]
sentinel_output = ["done"]
llm._add_chat_requests = Mock()
llm._run_engine = Mock(return_value=sentinel_output)
outputs = llm._run_chat(
messages=messages,
params=sampling_params,
output_type=object,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == sentinel_output
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_run_completion_forwards_mm_processor_kwargs() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"min_pixels": 4 * 28 * 28}
sampling_params = SamplingParams(max_tokens=1)
sentinel_output = ["done"]
llm._add_completion_requests = Mock()
llm._run_engine = Mock(return_value=sentinel_output)
outputs = llm._run_completion(
prompts=["prompt"],
params=sampling_params,
output_type=object,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == sentinel_output
assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"max_dynamic_patch": 4}
sampling_params = SamplingParams(max_tokens=1)
llm._params_to_seq = Mock(return_value=[sampling_params])
llm._lora_request_to_seq = Mock(return_value=[None])
llm._priority_to_seq = Mock(return_value=[0])
llm._preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
captured_prompts = []
def fake_render_and_add_requests(*, prompts, **_kwargs):
captured_prompts.extend(prompts)
return ["req-0"]
llm._render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
request_ids = llm._add_completion_requests(
prompts=["prompt"],
params=sampling_params,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert request_ids == ["req-0"]
llm._preprocess_cmpl_one.assert_called_once_with(
"prompt",
None,
mm_processor_kwargs=mm_processor_kwargs,
)
assert captured_prompts == [{"prompt_token_ids": [1]}]
def test_preprocess_cmpl_applies_mm_processor_kwargs_to_renderer(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
prompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
renderer = Mock()
renderer.default_cmpl_tok_params = Mock()
renderer.default_cmpl_tok_params.with_kwargs.return_value = "tok-params"
renderer.render_cmpl.return_value = ["engine-input"]
llm.renderer = renderer
monkeypatch.setattr(
"vllm.entrypoints.offline_utils.parse_model_prompt",
lambda _model_config, parsed_prompt: parsed_prompt,
)
outputs = llm._preprocess_cmpl(
[prompt],
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == ["engine-input"]
renderer.render_cmpl.assert_called_once_with(
[prompt],
"tok-params",
prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
)
def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
prompt = {
"prompt": "<image>",
"multi_modal_data": {"image": object()},
"mm_processor_kwargs": {"num_crops": 2},
}
renderer = Mock()
renderer.default_cmpl_tok_params = Mock()
renderer.default_cmpl_tok_params.with_kwargs.return_value = "tok-params"
renderer.render_cmpl.return_value = ["engine-input"]
llm.renderer = renderer
monkeypatch.setattr(
"vllm.entrypoints.offline_utils.parse_model_prompt",
lambda _model_config, parsed_prompt: parsed_prompt,
)
outputs = llm._preprocess_cmpl([prompt])
assert outputs == ["engine-input"]
renderer.render_cmpl.assert_called_once_with(
[prompt],
"tok-params",
prompt_extras=None,
)
def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
messages = [[{"role": "user", "content": "Describe this image."}]]
renderer = Mock()
renderer.tokenizer = object()
renderer.default_chat_tok_params = Mock()
renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
renderer.render_chat.return_value = (messages, ["engine-input"])
llm.renderer = renderer
outputs = llm._preprocess_chat(
messages,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == ["engine-input"]
call_args = renderer.render_chat.call_args
assert call_args.args[0] == messages
assert call_args.args[1].mm_processor_kwargs == mm_processor_kwargs
assert call_args.args[2] == "tok-params"
assert call_args.kwargs["prompt_extras"] == {
"mm_processor_kwargs": mm_processor_kwargs
}
def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
llm = _make_mock_llm()
messages = [[{"role": "user", "content": "Describe this image."}]]
renderer = Mock()
renderer.tokenizer = object()
renderer.default_chat_tok_params = Mock()
renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
renderer.render_chat.return_value = (messages, ["engine-input"])
llm.renderer = renderer
outputs = llm._preprocess_chat(messages)
assert outputs == ["engine-input"]
call_args = renderer.render_chat.call_args
assert call_args.args[0] == messages
assert call_args.args[1].mm_processor_kwargs is None
assert call_args.args[2] == "tok-params"
assert call_args.kwargs["prompt_extras"] is None