[CI] Consolidate multimodal entrypoint tests. (#44819)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -4,7 +4,6 @@ import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
|
||||
from vllm import LLM
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.sampling_params import SamplingParams
|
||||
@@ -76,47 +75,6 @@ def test_multi_chat(text_llm):
|
||||
assert len(outputs) == 2
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def vision_llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model="microsoft/Phi-3.5-vision-instruct",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
seed=0,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_urls", [[TEST_IMAGE_ASSETS[0], TEST_IMAGE_ASSETS[1]]], indirect=True
|
||||
)
|
||||
def test_chat_multi_image(vision_llm, image_urls: list[str]):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*(
|
||||
{"type": "image_url", "image_url": {"url": image_url}}
|
||||
for image_url in image_urls
|
||||
),
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
],
|
||||
}
|
||||
]
|
||||
outputs = vision_llm.chat(messages)
|
||||
assert len(outputs) >= 0
|
||||
|
||||
|
||||
def test_llm_chat_tokenization_no_double_bos(text_llm):
|
||||
"""
|
||||
LLM.chat() should not add special tokens when using chat templates.
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test that ``InputProcessor.inject_into_mm_cache()`` correctly injects
|
||||
pre-processed mm_kwargs into the processor cache and reports MM cache
|
||||
hit rate metrics accurately.
|
||||
|
||||
This is used by frameworks like Dynamo that run the HF processor on a
|
||||
frontend and transfer pre-processed mm_kwargs to the backend, avoiding
|
||||
redundant processing.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.renderers.params import ChatParams
|
||||
from vllm.v1.metrics import loggers as stat_loggers
|
||||
from vllm.v1.metrics.reader import Counter, Metric
|
||||
|
||||
|
||||
def _make_messages(image_url: str):
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _get_counter_value(metrics: list[Metric], name: str):
|
||||
metric = next(m for m in metrics if m.name == name)
|
||||
assert isinstance(metric, Counter)
|
||||
return metric.value
|
||||
|
||||
|
||||
def _get_mm_cache_stats(metrics: list[Metric]):
|
||||
mm_cache_queries = _get_counter_value(metrics, "vllm:mm_cache_queries")
|
||||
mm_cache_hits = _get_counter_value(metrics, "vllm:mm_cache_hits")
|
||||
return mm_cache_queries, mm_cache_hits
|
||||
|
||||
|
||||
def _get_mm_cache_log(llm: LLM, caplog_vllm: pytest.LogCaptureFixture) -> float:
|
||||
caplog_vllm.clear()
|
||||
with caplog_vllm.at_level(logging.INFO, logger=stat_loggers.__name__):
|
||||
llm.llm_engine.do_log_stats()
|
||||
|
||||
assert len(caplog_vllm.records) == 1
|
||||
msg = caplog_vllm.records[0].getMessage()
|
||||
|
||||
assert "MM cache hit rate" in msg
|
||||
match = re.search(r"MM cache hit rate: ([0-9.]+)%", msg)
|
||||
assert match is not None
|
||||
return float(match.group(1))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:2]], indirect=True)
|
||||
@pytest.mark.parametrize("mm_processor_cache_type", ["lru", "shm"])
|
||||
def test_inject_into_mm_cache(
|
||||
num_gpus_available,
|
||||
image_urls,
|
||||
mm_processor_cache_type,
|
||||
caplog_vllm,
|
||||
):
|
||||
"""Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
|
||||
the processor cache and MM cache hit metrics are updated correctly.
|
||||
|
||||
Steps:
|
||||
1. Two normal requests (same image) -> cache miss then hit (baseline)
|
||||
2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
|
||||
then generate with a pre-rendered input -> verifies injection works
|
||||
"""
|
||||
llm = LLM(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
disable_log_stats=False,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
mm_processor_cache_type=mm_processor_cache_type,
|
||||
)
|
||||
|
||||
# Step 1: Normal requests to populate the cache
|
||||
llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
|
||||
|
||||
llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 1)
|
||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(50.0)
|
||||
|
||||
# Step 2: Use a second image to get valid expanded tokens and
|
||||
# placeholder positions via the renderer.
|
||||
llm.chat(_make_messages(image_urls[1]))
|
||||
queries_before = _get_mm_cache_stats(llm.get_metrics())[0] # 3
|
||||
|
||||
renderer = llm.llm_engine.renderer
|
||||
cache = renderer.mm_processor_cache
|
||||
assert cache is not None, "Processor cache should be enabled"
|
||||
|
||||
_, eng_prompts = renderer.render_chat(
|
||||
[_make_messages(image_urls[1])],
|
||||
ChatParams(),
|
||||
)
|
||||
eng_input = eng_prompts[0]
|
||||
|
||||
# Inject pre-processed mm_kwargs with a NEW hash via public API
|
||||
new_mm_hash = "deadbeef" * 8
|
||||
mm_hashes = {"image": [new_mm_hash]}
|
||||
mm_kwargs = eng_input["mm_kwargs"]
|
||||
|
||||
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
|
||||
|
||||
# Build pre-rendered input (no externally_processed flag needed)
|
||||
pre_rendered_input = {
|
||||
"type": "multimodal",
|
||||
"prompt_token_ids": eng_input["prompt_token_ids"],
|
||||
"mm_kwargs": mm_kwargs,
|
||||
"mm_hashes": mm_hashes,
|
||||
"mm_placeholders": eng_input["mm_placeholders"],
|
||||
}
|
||||
|
||||
llm.generate(
|
||||
pre_rendered_input,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
)
|
||||
|
||||
# Verify cache was queried and injection happened
|
||||
queries_after = _get_mm_cache_stats(llm.get_metrics())[0]
|
||||
assert queries_after > queries_before, (
|
||||
"Cache should have been queried for the injected item"
|
||||
)
|
||||
mm_rate = _get_mm_cache_log(llm, caplog_vllm)
|
||||
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True)
|
||||
def test_inject_into_mm_cache_without_cache(
|
||||
num_gpus_available,
|
||||
image_urls,
|
||||
):
|
||||
"""Test that inject_into_mm_cache works gracefully when processor cache
|
||||
is disabled (mm_processor_cache_gb=0). Should not crash.
|
||||
"""
|
||||
llm = LLM(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
disable_log_stats=False,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
mm_processor_cache_gb=0,
|
||||
)
|
||||
|
||||
# Run a normal chat request first to warm up the model.
|
||||
llm.chat(_make_messages(image_urls[0]))
|
||||
|
||||
# Use the renderer to get a proper EngineInput with expanded tokens
|
||||
renderer = llm.llm_engine.renderer
|
||||
_, eng_prompts = renderer.render_chat(
|
||||
[_make_messages(image_urls[0])],
|
||||
ChatParams(),
|
||||
)
|
||||
eng_input = eng_prompts[0]
|
||||
|
||||
mm_hashes = {"image": ["abcd1234" * 8]}
|
||||
mm_kwargs = eng_input["mm_kwargs"]
|
||||
|
||||
# inject_into_mm_cache should not crash even without cache
|
||||
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
|
||||
|
||||
# Build and generate with pre-rendered input
|
||||
pre_rendered_input = {
|
||||
"type": "multimodal",
|
||||
"prompt_token_ids": eng_input["prompt_token_ids"],
|
||||
"mm_kwargs": mm_kwargs,
|
||||
"mm_hashes": mm_hashes,
|
||||
"mm_placeholders": eng_input["mm_placeholders"],
|
||||
}
|
||||
|
||||
result = llm.generate(
|
||||
pre_rendered_input,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
)
|
||||
assert len(result) == 1, "Should produce one output"
|
||||
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
|
||||
@@ -1,97 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
|
||||
from vllm import LLM
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.v1.metrics import loggers as stat_loggers
|
||||
from vllm.v1.metrics.reader import Counter, Metric
|
||||
|
||||
|
||||
def _make_messages(image_url: str) -> list[ChatCompletionMessageParam]:
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _get_counter_value(metrics: list[Metric], name: str):
|
||||
metric = next(m for m in metrics if m.name == name)
|
||||
assert isinstance(metric, Counter)
|
||||
return metric.value
|
||||
|
||||
|
||||
def _get_mm_cache_stats(metrics: list[Metric]):
|
||||
mm_cache_queries = _get_counter_value(metrics, "vllm:mm_cache_queries")
|
||||
mm_cache_hits = _get_counter_value(metrics, "vllm:mm_cache_hits")
|
||||
|
||||
return mm_cache_queries, mm_cache_hits
|
||||
|
||||
|
||||
def _get_mm_cache_log(llm: LLM, caplog_vllm: pytest.LogCaptureFixture) -> float:
|
||||
caplog_vllm.clear()
|
||||
with caplog_vllm.at_level(logging.INFO, logger=stat_loggers.__name__):
|
||||
llm.llm_engine.do_log_stats()
|
||||
|
||||
assert len(caplog_vllm.records) == 1
|
||||
msg = caplog_vllm.records[0].getMessage()
|
||||
|
||||
assert "MM cache hit rate" in msg
|
||||
match = re.search(r"MM cache hit rate: ([0-9.]+)%", msg)
|
||||
assert match is not None
|
||||
return float(match.group(1))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:2]], indirect=True)
|
||||
@pytest.mark.parametrize("mm_processor_cache_type", ["lru", "shm"])
|
||||
def test_mm_cache_stats(
|
||||
num_gpus_available,
|
||||
image_urls,
|
||||
mm_processor_cache_type,
|
||||
caplog_vllm,
|
||||
):
|
||||
llm = LLM(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
mm_processor_cache_type=mm_processor_cache_type,
|
||||
disable_log_stats=False,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
)
|
||||
|
||||
llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
|
||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
|
||||
|
||||
llm.chat(_make_messages(image_urls[1]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 0)
|
||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
|
||||
|
||||
llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (3, 1)
|
||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(33.3)
|
||||
|
||||
# NOTE: This only resets hit rate stats in CachingMetrics
|
||||
# The raw queries and hits counts remain unaffected
|
||||
llm.reset_mm_cache()
|
||||
|
||||
llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (4, 1)
|
||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
|
||||
|
||||
llm.chat(_make_messages(image_urls[1]))
|
||||
assert _get_mm_cache_stats(llm.get_metrics()) == (5, 1)
|
||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL = "llava-hf/llava-1.5-7b-hf"
|
||||
PROMPT = "USER: <image>\nDescribe this image briefly.\nASSISTANT:"
|
||||
TEXT_ONLY_PROMPT = "USER: What is 2 + 2?\nASSISTANT:"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
"""LLM with enable_mm_embeds=True and all modality limits zeroed out."""
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.8,
|
||||
enable_mm_embeds=True,
|
||||
limit_mm_per_prompt={"image": 0},
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_generate_with_embedding(llm: LLM):
|
||||
"""Pre-computed embedding produces tokens without hanging."""
|
||||
embedding = ImageAsset("stop_sign").image_embeds
|
||||
outputs = llm.generate(
|
||||
{"prompt": PROMPT, "multi_modal_data": {"image": embedding}},
|
||||
sampling_params=SamplingParams(max_tokens=32, temperature=0.0),
|
||||
)
|
||||
assert len(outputs) == 1
|
||||
assert len(outputs[0].outputs[0].text) > 0
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_raw_image_rejected(llm: LLM):
|
||||
"""Raw image input is still rejected when limit=0."""
|
||||
raw_image = ImageAsset("stop_sign").pil_image
|
||||
with pytest.raises(ValueError, match=r"At most 0 image\(s\)"):
|
||||
llm.generate(
|
||||
{"prompt": PROMPT, "multi_modal_data": {"image": raw_image}},
|
||||
sampling_params=SamplingParams(max_tokens=16),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_text_only_prompt(llm: LLM):
|
||||
"""Text-only prompts still work under this config."""
|
||||
outputs = llm.generate(
|
||||
TEXT_ONLY_PROMPT,
|
||||
sampling_params=SamplingParams(max_tokens=16, temperature=0.0),
|
||||
)
|
||||
assert len(outputs) == 1
|
||||
assert len(outputs[0].outputs[0].text) > 0
|
||||
@@ -1,288 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
|
||||
def _make_mock_llm() -> LLM:
|
||||
llm = object.__new__(LLM)
|
||||
llm.model_config = SimpleNamespace(
|
||||
runner_type="generate", enable_prompt_embeds=False
|
||||
)
|
||||
return llm
|
||||
|
||||
|
||||
def test_generate_forwards_mm_processor_kwargs() -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 4}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
llm._run_completion = Mock(return_value=["ok"])
|
||||
|
||||
outputs = llm.generate(
|
||||
"prompt",
|
||||
sampling_params=sampling_params,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == ["ok"]
|
||||
assert llm._run_completion.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_enqueue_forwards_mm_processor_kwargs() -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"do_resize": False}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
llm._add_completion_requests = Mock(return_value=["req-0"])
|
||||
|
||||
request_ids = llm.enqueue(
|
||||
"prompt",
|
||||
sampling_params=sampling_params,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert request_ids == ["req-0"]
|
||||
assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_chat_forwards_mm_processor_kwargs() -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"do_pan_and_scan": True}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
|
||||
llm._run_chat = Mock(return_value=["ok"])
|
||||
|
||||
outputs = llm.chat(
|
||||
messages,
|
||||
sampling_params=sampling_params,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == ["ok"]
|
||||
assert llm._run_chat.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_enqueue_chat_forwards_mm_processor_kwargs() -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"do_pan_and_scan": True}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
|
||||
llm._add_chat_requests = Mock(return_value=["req-0"])
|
||||
|
||||
request_ids = llm.enqueue_chat(
|
||||
messages,
|
||||
sampling_params=sampling_params,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert request_ids == ["req-0"]
|
||||
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_run_chat_forwards_mm_processor_kwargs() -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 8}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
sentinel_output = ["done"]
|
||||
|
||||
llm._add_chat_requests = Mock()
|
||||
llm._run_engine = Mock(return_value=sentinel_output)
|
||||
|
||||
outputs = llm._run_chat(
|
||||
messages=messages,
|
||||
params=sampling_params,
|
||||
output_type=object,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == sentinel_output
|
||||
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_run_completion_forwards_mm_processor_kwargs() -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"min_pixels": 4 * 28 * 28}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
sentinel_output = ["done"]
|
||||
|
||||
llm._add_completion_requests = Mock()
|
||||
llm._run_engine = Mock(return_value=sentinel_output)
|
||||
|
||||
outputs = llm._run_completion(
|
||||
prompts=["prompt"],
|
||||
params=sampling_params,
|
||||
output_type=object,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == sentinel_output
|
||||
assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"max_dynamic_patch": 4}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
llm._params_to_seq = Mock(return_value=[sampling_params])
|
||||
llm._lora_request_to_seq = Mock(return_value=[None])
|
||||
llm._priority_to_seq = Mock(return_value=[0])
|
||||
llm._preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
def fake_render_and_add_requests(*, prompts, **_kwargs):
|
||||
captured_prompts.extend(prompts)
|
||||
return ["req-0"]
|
||||
|
||||
llm._render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
|
||||
|
||||
request_ids = llm._add_completion_requests(
|
||||
prompts=["prompt"],
|
||||
params=sampling_params,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert request_ids == ["req-0"]
|
||||
llm._preprocess_cmpl_one.assert_called_once_with(
|
||||
"prompt",
|
||||
None,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
assert captured_prompts == [{"prompt_token_ids": [1]}]
|
||||
|
||||
|
||||
def test_preprocess_cmpl_applies_mm_processor_kwargs_to_renderer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 8}
|
||||
prompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
|
||||
|
||||
renderer = Mock()
|
||||
renderer.default_cmpl_tok_params = Mock()
|
||||
renderer.default_cmpl_tok_params.with_kwargs.return_value = "tok-params"
|
||||
renderer.render_cmpl.return_value = ["engine-input"]
|
||||
llm.renderer = renderer
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.offline_utils.parse_model_prompt",
|
||||
lambda _model_config, parsed_prompt: parsed_prompt,
|
||||
)
|
||||
|
||||
outputs = llm._preprocess_cmpl(
|
||||
[prompt],
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == ["engine-input"]
|
||||
renderer.render_cmpl.assert_called_once_with(
|
||||
[prompt],
|
||||
"tok-params",
|
||||
prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
|
||||
)
|
||||
|
||||
|
||||
def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
prompt = {
|
||||
"prompt": "<image>",
|
||||
"multi_modal_data": {"image": object()},
|
||||
"mm_processor_kwargs": {"num_crops": 2},
|
||||
}
|
||||
|
||||
renderer = Mock()
|
||||
renderer.default_cmpl_tok_params = Mock()
|
||||
renderer.default_cmpl_tok_params.with_kwargs.return_value = "tok-params"
|
||||
renderer.render_cmpl.return_value = ["engine-input"]
|
||||
llm.renderer = renderer
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.offline_utils.parse_model_prompt",
|
||||
lambda _model_config, parsed_prompt: parsed_prompt,
|
||||
)
|
||||
|
||||
outputs = llm._preprocess_cmpl([prompt])
|
||||
|
||||
assert outputs == ["engine-input"]
|
||||
renderer.render_cmpl.assert_called_once_with(
|
||||
[prompt],
|
||||
"tok-params",
|
||||
prompt_extras=None,
|
||||
)
|
||||
|
||||
|
||||
def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 8}
|
||||
messages = [[{"role": "user", "content": "Describe this image."}]]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
renderer.default_chat_tok_params = Mock()
|
||||
renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
|
||||
renderer.render_chat.return_value = (messages, ["engine-input"])
|
||||
llm.renderer = renderer
|
||||
|
||||
outputs = llm._preprocess_chat(
|
||||
messages,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == ["engine-input"]
|
||||
call_args = renderer.render_chat.call_args
|
||||
assert call_args.args[0] == messages
|
||||
assert call_args.args[1].mm_processor_kwargs == mm_processor_kwargs
|
||||
assert call_args.args[2] == "tok-params"
|
||||
assert call_args.kwargs["prompt_extras"] == {
|
||||
"mm_processor_kwargs": mm_processor_kwargs
|
||||
}
|
||||
|
||||
|
||||
def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
|
||||
llm = _make_mock_llm()
|
||||
messages = [[{"role": "user", "content": "Describe this image."}]]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
renderer.default_chat_tok_params = Mock()
|
||||
renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
|
||||
renderer.render_chat.return_value = (messages, ["engine-input"])
|
||||
llm.renderer = renderer
|
||||
|
||||
outputs = llm._preprocess_chat(messages)
|
||||
|
||||
assert outputs == ["engine-input"]
|
||||
call_args = renderer.render_chat.call_args
|
||||
assert call_args.args[0] == messages
|
||||
assert call_args.args[1].mm_processor_kwargs is None
|
||||
assert call_args.args[2] == "tok-params"
|
||||
assert call_args.kwargs["prompt_extras"] is None
|
||||
Reference in New Issue
Block a user