[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,192 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test that ``InputProcessor.inject_into_mm_cache()`` correctly injects
|
||||
pre-processed mm_kwargs into the processor cache and reports MM cache
|
||||
hit rate metrics accurately.
|
||||
|
||||
This is used by frameworks like Dynamo that run the HF processor on a
|
||||
frontend and transfer pre-processed mm_kwargs to the backend, avoiding
|
||||
redundant processing.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.renderers.params import ChatParams
|
||||
from vllm.v1.metrics import loggers as stat_loggers
|
||||
from vllm.v1.metrics.reader import Counter, Metric
|
||||
|
||||
|
||||
def _make_messages(image_url: str):
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _get_counter_value(metrics: list[Metric], name: str):
|
||||
metric = next(m for m in metrics if m.name == name)
|
||||
assert isinstance(metric, Counter)
|
||||
return metric.value
|
||||
|
||||
|
||||
def _get_mm_cache_stats(metrics: list[Metric]):
|
||||
mm_cache_queries = _get_counter_value(metrics, "vllm:mm_cache_queries")
|
||||
mm_cache_hits = _get_counter_value(metrics, "vllm:mm_cache_hits")
|
||||
return mm_cache_queries, mm_cache_hits
|
||||
|
||||
|
||||
def _get_mm_cache_log(llm: LLM, caplog_vllm: pytest.LogCaptureFixture) -> float:
|
||||
caplog_vllm.clear()
|
||||
with caplog_vllm.at_level(logging.INFO, logger=stat_loggers.__name__):
|
||||
llm.llm_engine.do_log_stats()
|
||||
|
||||
assert len(caplog_vllm.records) == 1
|
||||
msg = caplog_vllm.records[0].getMessage()
|
||||
|
||||
assert "MM cache hit rate" in msg
|
||||
match = re.search(r"MM cache hit rate: ([0-9.]+)%", msg)
|
||||
assert match is not None
|
||||
return float(match.group(1))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:2]], indirect=True)
|
||||
@pytest.mark.parametrize("mm_processor_cache_type", ["lru", "shm"])
|
||||
def test_inject_into_mm_cache(
|
||||
num_gpus_available,
|
||||
image_urls,
|
||||
mm_processor_cache_type,
|
||||
caplog_vllm,
|
||||
vllm_runner,
|
||||
):
|
||||
"""Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
|
||||
the processor cache and MM cache hit metrics are updated correctly.
|
||||
|
||||
Steps:
|
||||
1. Two normal requests (same image) -> cache miss then hit (baseline)
|
||||
2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
|
||||
then generate with a pre-rendered input -> verifies injection works
|
||||
"""
|
||||
with vllm_runner(
|
||||
"llava-hf/llava-1.5-7b-hf",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
disable_log_stats=False,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
mm_processor_cache_type=mm_processor_cache_type,
|
||||
) as runner:
|
||||
# Step 1: Normal requests to populate the cache
|
||||
runner.llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
|
||||
|
||||
runner.llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 1)
|
||||
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(50.0)
|
||||
|
||||
# Step 2: Use a second image to get valid expanded tokens and
|
||||
# placeholder positions via the renderer.
|
||||
runner.llm.chat(_make_messages(image_urls[1]))
|
||||
queries_before = _get_mm_cache_stats(runner.llm.get_metrics())[0] # 3
|
||||
|
||||
renderer = runner.llm.llm_engine.renderer
|
||||
cache = renderer.mm_processor_cache
|
||||
assert cache is not None, "Processor cache should be enabled"
|
||||
|
||||
_, eng_prompts = renderer.render_chat(
|
||||
[_make_messages(image_urls[1])],
|
||||
ChatParams(),
|
||||
)
|
||||
eng_input = eng_prompts[0]
|
||||
|
||||
# Inject pre-processed mm_kwargs with a NEW hash via public API
|
||||
new_mm_hash = "deadbeef" * 8
|
||||
mm_hashes = {"image": [new_mm_hash]}
|
||||
mm_kwargs = eng_input["mm_kwargs"]
|
||||
|
||||
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
|
||||
|
||||
# Build pre-rendered input (no externally_processed flag needed)
|
||||
pre_rendered_input = {
|
||||
"type": "multimodal",
|
||||
"prompt_token_ids": eng_input["prompt_token_ids"],
|
||||
"mm_kwargs": mm_kwargs,
|
||||
"mm_hashes": mm_hashes,
|
||||
"mm_placeholders": eng_input["mm_placeholders"],
|
||||
}
|
||||
|
||||
runner.llm.generate(
|
||||
pre_rendered_input,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
)
|
||||
|
||||
# Verify cache was queried and injection happened
|
||||
queries_after = _get_mm_cache_stats(runner.llm.get_metrics())[0]
|
||||
assert queries_after > queries_before, (
|
||||
"Cache should have been queried for the injected item"
|
||||
)
|
||||
mm_rate = _get_mm_cache_log(runner.llm, caplog_vllm)
|
||||
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True)
|
||||
def test_inject_into_mm_cache_without_cache(
|
||||
num_gpus_available,
|
||||
image_urls,
|
||||
vllm_runner,
|
||||
):
|
||||
"""Test that inject_into_mm_cache works gracefully when processor cache
|
||||
is disabled (mm_processor_cache_gb=0). Should not crash.
|
||||
"""
|
||||
with vllm_runner(
|
||||
"llava-hf/llava-1.5-7b-hf",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
disable_log_stats=False,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
mm_processor_cache_gb=0,
|
||||
) as runner:
|
||||
# Run a normal chat request first to warm up the model.
|
||||
runner.llm.chat(_make_messages(image_urls[0]))
|
||||
|
||||
# Use the renderer to get a proper EngineInput with expanded tokens
|
||||
renderer = runner.llm.llm_engine.renderer
|
||||
_, eng_prompts = renderer.render_chat(
|
||||
[_make_messages(image_urls[0])],
|
||||
ChatParams(),
|
||||
)
|
||||
eng_input = eng_prompts[0]
|
||||
|
||||
mm_hashes = {"image": ["abcd1234" * 8]}
|
||||
mm_kwargs = eng_input["mm_kwargs"]
|
||||
|
||||
# inject_into_mm_cache should not crash even without cache
|
||||
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
|
||||
|
||||
# Build and generate with pre-rendered input
|
||||
pre_rendered_input = {
|
||||
"type": "multimodal",
|
||||
"prompt_token_ids": eng_input["prompt_token_ids"],
|
||||
"mm_kwargs": mm_kwargs,
|
||||
"mm_hashes": mm_hashes,
|
||||
"mm_placeholders": eng_input["mm_placeholders"],
|
||||
}
|
||||
|
||||
result = runner.llm.generate(
|
||||
pre_rendered_input,
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
)
|
||||
assert len(result) == 1, "Should produce one output"
|
||||
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
|
||||
@@ -1,97 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
||||
from vllm import LLM
|
||||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.v1.metrics import loggers as stat_loggers
|
||||
from vllm.v1.metrics.reader import Counter, Metric
|
||||
|
||||
|
||||
def _make_messages(image_url: str) -> list[ChatCompletionMessageParam]:
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _get_counter_value(metrics: list[Metric], name: str):
|
||||
metric = next(m for m in metrics if m.name == name)
|
||||
assert isinstance(metric, Counter)
|
||||
return metric.value
|
||||
|
||||
|
||||
def _get_mm_cache_stats(metrics: list[Metric]):
|
||||
mm_cache_queries = _get_counter_value(metrics, "vllm:mm_cache_queries")
|
||||
mm_cache_hits = _get_counter_value(metrics, "vllm:mm_cache_hits")
|
||||
|
||||
return mm_cache_queries, mm_cache_hits
|
||||
|
||||
|
||||
def _get_mm_cache_log(llm: LLM, caplog_vllm: pytest.LogCaptureFixture) -> float:
|
||||
caplog_vllm.clear()
|
||||
with caplog_vllm.at_level(logging.INFO, logger=stat_loggers.__name__):
|
||||
llm.llm_engine.do_log_stats()
|
||||
|
||||
assert len(caplog_vllm.records) == 1
|
||||
msg = caplog_vllm.records[0].getMessage()
|
||||
|
||||
assert "MM cache hit rate" in msg
|
||||
match = re.search(r"MM cache hit rate: ([0-9.]+)%", msg)
|
||||
assert match is not None
|
||||
return float(match.group(1))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:2]], indirect=True)
|
||||
@pytest.mark.parametrize("mm_processor_cache_type", ["lru", "shm"])
|
||||
def test_mm_cache_stats(
|
||||
num_gpus_available,
|
||||
image_urls,
|
||||
mm_processor_cache_type,
|
||||
caplog_vllm,
|
||||
vllm_runner,
|
||||
):
|
||||
with vllm_runner(
|
||||
"llava-hf/llava-1.5-7b-hf",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=5,
|
||||
enforce_eager=True,
|
||||
mm_processor_cache_type=mm_processor_cache_type,
|
||||
disable_log_stats=False,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
) as runner:
|
||||
runner.llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
|
||||
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
|
||||
|
||||
runner.llm.chat(_make_messages(image_urls[1]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 0)
|
||||
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
|
||||
|
||||
runner.llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (3, 1)
|
||||
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(33.3)
|
||||
|
||||
# NOTE: This only resets hit rate stats in CachingMetrics
|
||||
# The raw queries and hits counts remain unaffected
|
||||
runner.llm.reset_mm_cache()
|
||||
|
||||
runner.llm.chat(_make_messages(image_urls[0]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (4, 1)
|
||||
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
|
||||
|
||||
runner.llm.chat(_make_messages(image_urls[1]))
|
||||
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (5, 1)
|
||||
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
|
||||
@@ -1,64 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
MODEL = "llava-hf/llava-1.5-7b-hf"
|
||||
PROMPT = "USER: <image>\nDescribe this image briefly.\nASSISTANT:"
|
||||
TEXT_ONLY_PROMPT = "USER: What is 2 + 2?\nASSISTANT:"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm(vllm_runner):
|
||||
"""LLM with enable_mm_embeds=True and all modality limits zeroed out."""
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.8,
|
||||
enable_mm_embeds=True,
|
||||
limit_mm_per_prompt={"image": 0},
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_generate_with_embedding(llm: LLM):
|
||||
"""Pre-computed embedding produces tokens without hanging."""
|
||||
embedding = ImageAsset("stop_sign").image_embeds
|
||||
outputs = llm.generate(
|
||||
{"prompt": PROMPT, "multi_modal_data": {"image": embedding}},
|
||||
sampling_params=SamplingParams(max_tokens=32, temperature=0.0),
|
||||
)
|
||||
assert len(outputs) == 1
|
||||
assert len(outputs[0].outputs[0].text) > 0
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_raw_image_rejected(llm: LLM):
|
||||
"""Raw image input is still rejected when limit=0."""
|
||||
raw_image = ImageAsset("stop_sign").pil_image
|
||||
with pytest.raises(VLLMValidationError, match=r"At most 0 image\(s\)"):
|
||||
llm.generate(
|
||||
{"prompt": PROMPT, "multi_modal_data": {"image": raw_image}},
|
||||
sampling_params=SamplingParams(max_tokens=16),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_text_only_prompt(llm: LLM):
|
||||
"""Text-only prompts still work under this config."""
|
||||
outputs = llm.generate(
|
||||
TEXT_ONLY_PROMPT,
|
||||
sampling_params=SamplingParams(max_tokens=16, temperature=0.0),
|
||||
)
|
||||
assert len(outputs) == 1
|
||||
assert len(outputs[0].outputs[0].text) > 0
|
||||
@@ -1,447 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||||
|
||||
|
||||
def _make_mock_llm() -> LLM:
|
||||
llm = object.__new__(LLM)
|
||||
llm.chat_template = None
|
||||
llm.model_config = SimpleNamespace(
|
||||
runner_type="generate", enable_prompt_embeds=False
|
||||
)
|
||||
return llm
|
||||
|
||||
|
||||
def test_generate_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 4}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
_run_completion = Mock(return_value=["ok"])
|
||||
monkeypatch.setattr(llm, "_run_completion", _run_completion)
|
||||
|
||||
outputs = llm.generate(
|
||||
"prompt",
|
||||
sampling_params=sampling_params,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == ["ok"]
|
||||
assert _run_completion.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_enqueue_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"do_resize": False}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
_add_completion_requests = Mock(return_value=["req-0"])
|
||||
monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests)
|
||||
|
||||
request_ids = llm.enqueue(
|
||||
"prompt",
|
||||
sampling_params=sampling_params,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert request_ids == ["req-0"]
|
||||
assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"do_pan_and_scan": True}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
|
||||
|
||||
_run_chat = Mock(return_value=["ok"])
|
||||
monkeypatch.setattr(llm, "_run_chat", _run_chat)
|
||||
|
||||
outputs = llm.chat(
|
||||
messages,
|
||||
sampling_params=sampling_params,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == ["ok"]
|
||||
assert _run_chat.call_args.kwargs["mm_processor_kwargs"] == mm_processor_kwargs
|
||||
|
||||
|
||||
def test_enqueue_chat_forwards_mm_processor_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"do_pan_and_scan": True}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
|
||||
|
||||
_add_chat_requests = Mock(return_value=["req-0"])
|
||||
monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests)
|
||||
|
||||
request_ids = llm.enqueue_chat(
|
||||
messages,
|
||||
sampling_params=sampling_params,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert request_ids == ["req-0"]
|
||||
assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_run_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 8}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
|
||||
sentinel_output = ["done"]
|
||||
|
||||
_add_chat_requests = Mock()
|
||||
monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests)
|
||||
_run_engine = Mock(return_value=sentinel_output)
|
||||
monkeypatch.setattr(llm, "_run_engine", _run_engine)
|
||||
|
||||
outputs = llm._run_chat(
|
||||
messages=messages,
|
||||
params=sampling_params,
|
||||
output_type=RequestOutput,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == sentinel_output
|
||||
assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_run_completion_forwards_mm_processor_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"min_pixels": 4 * 28 * 28}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
sentinel_output = ["done"]
|
||||
|
||||
_add_completion_requests = Mock()
|
||||
monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests)
|
||||
_run_engine = Mock(return_value=sentinel_output)
|
||||
monkeypatch.setattr(llm, "_run_engine", _run_engine)
|
||||
|
||||
outputs = llm._run_completion(
|
||||
prompts=["prompt"],
|
||||
params=sampling_params,
|
||||
output_type=RequestOutput,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == sentinel_output
|
||||
assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_add_completion_requests_forwards_mm_processor_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"max_dynamic_patch": 4}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
_params_to_seq = Mock(return_value=[sampling_params])
|
||||
monkeypatch.setattr(llm, "_params_to_seq", _params_to_seq)
|
||||
_lora_request_to_seq = Mock(return_value=[None])
|
||||
monkeypatch.setattr(llm, "_lora_request_to_seq", _lora_request_to_seq)
|
||||
_priority_to_seq = Mock(return_value=[0])
|
||||
monkeypatch.setattr(llm, "_priority_to_seq", _priority_to_seq)
|
||||
_preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
|
||||
monkeypatch.setattr(llm, "_preprocess_cmpl_one", _preprocess_cmpl_one)
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
def fake_render_and_add_requests(*, prompts, **_kwargs):
|
||||
captured_prompts.extend(prompts)
|
||||
return ["req-0"]
|
||||
|
||||
_render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
|
||||
monkeypatch.setattr(llm, "_render_and_add_requests", _render_and_add_requests)
|
||||
|
||||
request_ids = llm._add_completion_requests(
|
||||
prompts=["prompt"],
|
||||
params=sampling_params,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert request_ids == ["req-0"]
|
||||
_preprocess_cmpl_one.assert_called_once_with(
|
||||
"prompt",
|
||||
None,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
assert captured_prompts == [{"prompt_token_ids": [1]}]
|
||||
|
||||
|
||||
def test_preprocess_cmpl_applies_mm_processor_kwargs_to_renderer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 8}
|
||||
prompt: TextPrompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
|
||||
|
||||
renderer = Mock()
|
||||
renderer.default_cmpl_tok_params = Mock()
|
||||
renderer.default_cmpl_tok_params.with_kwargs.return_value = "tok-params"
|
||||
renderer.render_cmpl.return_value = ["engine-input"]
|
||||
llm.renderer = renderer
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.common.offline.parse_model_prompt",
|
||||
lambda _model_config, parsed_prompt: parsed_prompt,
|
||||
)
|
||||
|
||||
outputs = llm._preprocess_cmpl(
|
||||
[prompt],
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == ["engine-input"]
|
||||
renderer.render_cmpl.assert_called_once_with(
|
||||
[prompt],
|
||||
"tok-params",
|
||||
prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
|
||||
)
|
||||
|
||||
|
||||
def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
prompt: TextPrompt = {
|
||||
"prompt": "<image>",
|
||||
"multi_modal_data": {"image": object()},
|
||||
"mm_processor_kwargs": {"num_crops": 2},
|
||||
}
|
||||
|
||||
renderer = Mock()
|
||||
renderer.default_cmpl_tok_params = Mock()
|
||||
renderer.default_cmpl_tok_params.with_kwargs.return_value = "tok-params"
|
||||
renderer.render_cmpl.return_value = ["engine-input"]
|
||||
llm.renderer = renderer
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.common.offline.parse_model_prompt",
|
||||
lambda _model_config, parsed_prompt: parsed_prompt,
|
||||
)
|
||||
|
||||
outputs = llm._preprocess_cmpl([prompt])
|
||||
|
||||
assert outputs == ["engine-input"]
|
||||
renderer.render_cmpl.assert_called_once_with(
|
||||
[prompt],
|
||||
"tok-params",
|
||||
prompt_extras=None,
|
||||
)
|
||||
|
||||
|
||||
def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 8}
|
||||
messages: list[list[ChatCompletionMessageParam]] = [
|
||||
[{"role": "user", "content": "Describe this image."}]
|
||||
]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
renderer.default_chat_tok_params = Mock()
|
||||
renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
|
||||
renderer.render_chat.return_value = (messages, ["engine-input"])
|
||||
llm.renderer = renderer
|
||||
|
||||
outputs = llm._preprocess_chat(
|
||||
messages,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == ["engine-input"]
|
||||
call_args = renderer.render_chat.call_args
|
||||
assert call_args.args[0] == messages
|
||||
assert call_args.args[1].mm_processor_kwargs == mm_processor_kwargs
|
||||
assert call_args.args[2] == "tok-params"
|
||||
assert call_args.kwargs["prompt_extras"] == {
|
||||
"mm_processor_kwargs": mm_processor_kwargs
|
||||
}
|
||||
|
||||
|
||||
def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
|
||||
llm = _make_mock_llm()
|
||||
messages: list[list[ChatCompletionMessageParam]] = [
|
||||
[{"role": "user", "content": "Describe this image."}]
|
||||
]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
renderer.default_chat_tok_params = Mock()
|
||||
renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
|
||||
renderer.render_chat.return_value = (messages, ["engine-input"])
|
||||
llm.renderer = renderer
|
||||
|
||||
outputs = llm._preprocess_chat(messages)
|
||||
|
||||
assert outputs == ["engine-input"]
|
||||
call_args = renderer.render_chat.call_args
|
||||
assert call_args.args[0] == messages
|
||||
assert call_args.args[1].mm_processor_kwargs is None
|
||||
assert call_args.args[2] == "tok-params"
|
||||
assert call_args.kwargs["prompt_extras"] is None
|
||||
|
||||
|
||||
def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None:
|
||||
# Matches `ChatCompletionRequest.add_special_tokens` on the server.
|
||||
llm = _make_mock_llm()
|
||||
messages: list[list[ChatCompletionMessageParam]] = [
|
||||
[{"role": "user", "content": "hi"}]
|
||||
]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
renderer.default_chat_tok_params = Mock()
|
||||
renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
|
||||
renderer.render_chat.return_value = (messages, ["engine-input"])
|
||||
llm.renderer = renderer
|
||||
|
||||
llm._preprocess_chat(messages)
|
||||
|
||||
renderer.default_chat_tok_params.with_kwargs.assert_called_once_with(
|
||||
add_special_tokens=False
|
||||
)
|
||||
|
||||
|
||||
def test_preprocess_chat_tokenization_kwargs_override_add_special_tokens() -> None:
|
||||
llm = _make_mock_llm()
|
||||
messages: list[list[ChatCompletionMessageParam]] = [
|
||||
[{"role": "user", "content": "hi"}]
|
||||
]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
renderer.default_chat_tok_params = Mock()
|
||||
renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
|
||||
renderer.render_chat.return_value = (messages, ["engine-input"])
|
||||
llm.renderer = renderer
|
||||
|
||||
llm._preprocess_chat(
|
||||
messages,
|
||||
tokenization_kwargs={"add_special_tokens": True, "truncate_prompt_tokens": 8},
|
||||
)
|
||||
|
||||
renderer.default_chat_tok_params.with_kwargs.assert_called_once_with(
|
||||
add_special_tokens=True, truncate_prompt_tokens=8
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llava_llm():
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
|
||||
# A real multimodal model (Llama tokenizer, BOS=1) so that the real
|
||||
# processor default (`add_special_tokens=True`) is in play. Only
|
||||
# config/tokenizer/processor files are fetched, no weights.
|
||||
model_config = ModelConfig(model="llava-hf/llava-1.5-7b-hf", max_model_len=128)
|
||||
renderer = HfRenderer(
|
||||
VllmConfig(model_config=model_config),
|
||||
cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
assert renderer.default_chat_tok_params.add_special_tokens is True
|
||||
|
||||
llm = _make_mock_llm()
|
||||
# `_preprocess_cmpl` parses prompts against the real model config.
|
||||
llm.model_config = model_config
|
||||
llm.renderer = renderer
|
||||
return llm
|
||||
|
||||
|
||||
class TestChatAddSpecialTokensDefault:
|
||||
"""`LLM.chat()` renders the chat template to text and then tokenizes it.
|
||||
The multimodal processor default is `add_special_tokens=True`, so a
|
||||
template that emits `bos_token` used to get a second BOS from the
|
||||
tokenizer. `_preprocess_chat` now defaults `add_special_tokens=False`
|
||||
like the online chat API does (`ChatCompletionRequest`), unless the
|
||||
caller overrides it via `tokenization_kwargs`.
|
||||
"""
|
||||
|
||||
# Mirrors the Gemma 3 chat template and the bundled deepseek_vl2 /
|
||||
# deepseek_ocr templates, whose rendered output starts with the BOS token.
|
||||
BOS_TEMPLATE = (
|
||||
"{{ bos_token }}{% for m in messages %}{{ m['content'] }}{% endfor %}"
|
||||
)
|
||||
MESSAGES = [[{"role": "user", "content": "hi"}]]
|
||||
|
||||
def _chat_token_ids(self, llm, **kwargs):
|
||||
(engine_input,) = llm._preprocess_chat(
|
||||
self.MESSAGES,
|
||||
chat_template=self.BOS_TEMPLATE,
|
||||
add_generation_prompt=False,
|
||||
**kwargs,
|
||||
)
|
||||
return engine_input["prompt_token_ids"]
|
||||
|
||||
def test_chat_does_not_duplicate_template_bos(self, llava_llm):
|
||||
bos = llava_llm.renderer.tokenizer.bos_token_id
|
||||
prompt_token_ids = self._chat_token_ids(llava_llm)
|
||||
|
||||
assert prompt_token_ids[0] == bos
|
||||
assert prompt_token_ids.count(bos) == 1
|
||||
|
||||
def test_chat_matches_online_chat_api(self, llava_llm):
|
||||
from vllm.renderers.params import ChatParams, TokenizeParams
|
||||
|
||||
# `ChatCompletionRequest.add_special_tokens` defaults to `False`.
|
||||
online_tok_params = TokenizeParams(
|
||||
max_total_tokens=llava_llm.renderer.model_config.max_model_len,
|
||||
add_special_tokens=False,
|
||||
)
|
||||
chat_params = ChatParams(
|
||||
chat_template=self.BOS_TEMPLATE,
|
||||
chat_template_kwargs=dict(tokenize=False, add_generation_prompt=False),
|
||||
)
|
||||
_, (online_input,) = llava_llm.renderer.render_chat(
|
||||
self.MESSAGES, chat_params, online_tok_params
|
||||
)
|
||||
|
||||
assert self._chat_token_ids(llava_llm) == online_input["prompt_token_ids"]
|
||||
|
||||
def test_explicit_tokenization_kwargs_override_default(self, llava_llm):
|
||||
bos = llava_llm.renderer.tokenizer.bos_token_id
|
||||
prompt_token_ids = self._chat_token_ids(
|
||||
llava_llm, tokenization_kwargs={"add_special_tokens": True}
|
||||
)
|
||||
|
||||
assert prompt_token_ids.count(bos) == 2
|
||||
|
||||
def test_generate_keeps_processor_default(self, llava_llm):
|
||||
# Raw prompts have no chat template to emit BOS, so `LLM.generate()`
|
||||
# must keep letting the tokenizer add it.
|
||||
bos = llava_llm.renderer.tokenizer.bos_token_id
|
||||
(engine_input,) = llava_llm._preprocess_cmpl(["hi"])
|
||||
|
||||
assert engine_input["prompt_token_ids"][0] == bos
|
||||
@@ -1,515 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.assets.base import VLLM_S3_BUCKET_URL
|
||||
from vllm.multimodal.utils import encode_video_url, fetch_video
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL_NAME = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
MAXIMUM_VIDEOS = 3
|
||||
|
||||
TEST_VIDEO_URLS = [
|
||||
f"{VLLM_S3_BUCKET_URL}/multimodal_asset/slow_traffic_small.mp4",
|
||||
f"{VLLM_S3_BUCKET_URL}/multimodal_asset/vtest.avi",
|
||||
f"{VLLM_S3_BUCKET_URL}/multimodal_asset/Megamind.avi",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"generate",
|
||||
"--max-model-len",
|
||||
"32768",
|
||||
"--max-num-seqs",
|
||||
"2",
|
||||
"--enforce-eager",
|
||||
"--trust-remote-code",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"video": MAXIMUM_VIDEOS}),
|
||||
"--media-io-kwargs",
|
||||
json.dumps({"video": {"num_frames": 32}}),
|
||||
"--trust-request-mm-kwargs",
|
||||
]
|
||||
|
||||
# ROCm: Allow more time to download videos from external sources.
|
||||
env_overrides = {}
|
||||
if current_platform.is_rocm():
|
||||
env_overrides = {
|
||||
"VLLM_VIDEO_FETCH_TIMEOUT": "120",
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_overrides) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def url_encoded_video() -> dict[str, str]:
|
||||
return {
|
||||
video_url: encode_video_url(fetch_video(video_url)[0])
|
||||
for video_url in TEST_VIDEO_URLS
|
||||
}
|
||||
|
||||
|
||||
def assert_non_empty_content(chat_completion, *, context: str = "") -> str:
|
||||
"""Assert the first choice has non-empty string content; return it."""
|
||||
prefix = f"[{context}] " if context else ""
|
||||
choice = chat_completion.choices[0]
|
||||
content = choice.message.content
|
||||
|
||||
assert content is not None, (
|
||||
f"{prefix}Expected non-None content but got None. "
|
||||
f"finish_reason={choice.finish_reason!r}, "
|
||||
f"full message={choice.message!r}, "
|
||||
f"usage={chat_completion.usage!r}"
|
||||
)
|
||||
assert isinstance(content, str), (
|
||||
f"{prefix}Expected str content, got {type(content).__name__}: {content!r}"
|
||||
)
|
||||
assert len(content) > 0, (
|
||||
f"{prefix}Expected non-empty content but got empty string. "
|
||||
f"finish_reason={choice.finish_reason!r}, "
|
||||
f"full message={choice.message!r}, "
|
||||
f"usage={chat_completion.usage!r}"
|
||||
)
|
||||
return content
|
||||
|
||||
|
||||
def describe_video_messages(
|
||||
video_url: str | None, *, extra_video_fields: dict | None = None
|
||||
) -> list[dict]:
|
||||
"""Build the system + user messages used by the completions-with-video
|
||||
family of tests. *extra_video_fields* is merged into the top-level
|
||||
video content block (for uuid / bad-key tests).
|
||||
|
||||
Pass ``None`` for *video_url* to omit media (uuid-backed cache hits).
|
||||
"""
|
||||
video_block: dict = {
|
||||
"type": "video_url",
|
||||
"video_url": {} if video_url is None else {"url": video_url},
|
||||
}
|
||||
if extra_video_fields:
|
||||
video_block.update(extra_video_fields)
|
||||
|
||||
return [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this video."},
|
||||
video_block,
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
async def complete_and_check(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
messages: list[dict],
|
||||
*,
|
||||
context: str,
|
||||
max_completion_tokens: int = 50,
|
||||
temperature: float = 0.0,
|
||||
) -> str:
|
||||
"""Run a chat completion and assert the output is non-empty."""
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
temperature=temperature,
|
||||
)
|
||||
return assert_non_empty_content(chat_completion, context=context)
|
||||
|
||||
|
||||
def dummy_messages_from_video_url(
|
||||
video_urls: str | list[str],
|
||||
content_text: str = "What's in this video?",
|
||||
):
|
||||
if isinstance(video_urls, str):
|
||||
video_urls = [video_urls]
|
||||
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*(
|
||||
{"type": "video_url", "video_url": {"url": video_url}}
|
||||
for video_url in video_urls
|
||||
),
|
||||
{"type": "text", "text": content_text},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
|
||||
async def test_single_chat_session_video(
|
||||
client: openai.AsyncOpenAI, model_name: str, video_url: str
|
||||
):
|
||||
messages = dummy_messages_from_video_url(video_url)
|
||||
|
||||
# test single completion
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
logprobs=True,
|
||||
temperature=0.0,
|
||||
top_logprobs=5,
|
||||
)
|
||||
assert len(chat_completion.choices) == 1
|
||||
|
||||
choice = chat_completion.choices[0]
|
||||
assert choice.finish_reason == "length"
|
||||
assert chat_completion.usage == openai.types.CompletionUsage(
|
||||
completion_tokens=10, prompt_tokens=6287, total_tokens=6297
|
||||
)
|
||||
|
||||
message = choice.message
|
||||
message = chat_completion.choices[0].message
|
||||
assert message.content is not None and len(message.content) >= 10
|
||||
assert message.role == "assistant"
|
||||
messages.append({"role": "assistant", "content": message.content})
|
||||
|
||||
# test multi-turn dialogue
|
||||
messages.append({"role": "user", "content": "express your result in json"})
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
)
|
||||
message = chat_completion.choices[0].message
|
||||
assert message.content is not None and len(message.content) >= 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("video_url", [TEST_VIDEO_URLS[0]])
|
||||
async def test_request_media_io_kwargs_override_uses_fewer_video_frames(
|
||||
client: openai.AsyncOpenAI, model_name: str, video_url: str
|
||||
):
|
||||
messages = dummy_messages_from_video_url(video_url)
|
||||
|
||||
default_resp = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=1,
|
||||
temperature=0.0,
|
||||
)
|
||||
override_resp = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=1,
|
||||
temperature=0.0,
|
||||
extra_body={
|
||||
"media_io_kwargs": {
|
||||
"video": {
|
||||
"num_frames": 4,
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert default_resp.usage is not None
|
||||
assert override_resp.usage is not None
|
||||
assert override_resp.usage.prompt_tokens < default_resp.usage.prompt_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("video_url", [TEST_VIDEO_URLS[0]])
|
||||
async def test_invalid_num_frames_request_recoverable(
|
||||
client: openai.AsyncOpenAI, model_name: str, video_url: str
|
||||
):
|
||||
messages = dummy_messages_from_video_url(video_url)
|
||||
|
||||
with pytest.raises((openai.BadRequestError, openai.APIStatusError)):
|
||||
await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=1,
|
||||
temperature=0.0,
|
||||
extra_body={
|
||||
"media_io_kwargs": {
|
||||
"video": {
|
||||
"num_frames": "invalid",
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
# Server should still handle subsequent requests after the failed one.
|
||||
recovery_resp = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=1,
|
||||
temperature=0.0,
|
||||
)
|
||||
recovery_msg = recovery_resp.choices[0].message
|
||||
assert recovery_msg.content is not None and len(recovery_msg.content) >= 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
|
||||
async def test_error_on_invalid_video_url_type(
|
||||
client: openai.AsyncOpenAI, model_name: str, video_url: str
|
||||
):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "video_url", "video_url": video_url},
|
||||
{"type": "text", "text": "What's in this video?"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
# video_url should be a dict {"url": "some url"}, not directly a string
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
_ = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
|
||||
async def test_single_chat_session_video_beamsearch(
|
||||
client: openai.AsyncOpenAI, model_name: str, video_url: str
|
||||
):
|
||||
messages = dummy_messages_from_video_url(video_url)
|
||||
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
n=2,
|
||||
max_completion_tokens=10,
|
||||
logprobs=True,
|
||||
top_logprobs=5,
|
||||
extra_body=dict(use_beam_search=True),
|
||||
)
|
||||
assert len(chat_completion.choices) == 2
|
||||
assert (
|
||||
chat_completion.choices[0].message.content
|
||||
!= chat_completion.choices[1].message.content
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
|
||||
async def test_single_chat_session_video_base64encoded(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
video_url: str,
|
||||
url_encoded_video: dict[str, str],
|
||||
):
|
||||
messages = dummy_messages_from_video_url(url_encoded_video[video_url])
|
||||
|
||||
# test single completion
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
logprobs=True,
|
||||
temperature=0.0,
|
||||
top_logprobs=5,
|
||||
)
|
||||
assert len(chat_completion.choices) == 1
|
||||
|
||||
choice = chat_completion.choices[0]
|
||||
assert choice.finish_reason == "length"
|
||||
assert chat_completion.usage == openai.types.CompletionUsage(
|
||||
completion_tokens=10, prompt_tokens=6287, total_tokens=6297
|
||||
)
|
||||
|
||||
message = choice.message
|
||||
message = chat_completion.choices[0].message
|
||||
assert message.content is not None and len(message.content) >= 10
|
||||
assert message.role == "assistant"
|
||||
messages.append({"role": "assistant", "content": message.content})
|
||||
|
||||
# test multi-turn dialogue
|
||||
messages.append({"role": "user", "content": "express your result in json"})
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
)
|
||||
message = chat_completion.choices[0].message
|
||||
assert message.content is not None and len(message.content) >= 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
|
||||
async def test_single_chat_session_video_base64encoded_beamsearch(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
video_url: str,
|
||||
url_encoded_video: dict[str, str],
|
||||
):
|
||||
messages = dummy_messages_from_video_url(url_encoded_video[video_url])
|
||||
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
n=2,
|
||||
max_completion_tokens=10,
|
||||
extra_body=dict(use_beam_search=True),
|
||||
)
|
||||
assert len(chat_completion.choices) == 2
|
||||
assert (
|
||||
chat_completion.choices[0].message.content
|
||||
!= chat_completion.choices[1].message.content
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
|
||||
async def test_chat_streaming_video(
|
||||
client: openai.AsyncOpenAI, model_name: str, video_url: str
|
||||
):
|
||||
messages = dummy_messages_from_video_url(video_url)
|
||||
|
||||
# test single completion
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
)
|
||||
output = chat_completion.choices[0].message.content
|
||||
stop_reason = chat_completion.choices[0].finish_reason
|
||||
|
||||
# test streaming
|
||||
stream = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
)
|
||||
chunks: list[str] = []
|
||||
finish_reason_count = 0
|
||||
async for chunk in stream:
|
||||
delta = chunk.choices[0].delta
|
||||
if delta.role:
|
||||
assert delta.role == "assistant"
|
||||
if delta.content:
|
||||
chunks.append(delta.content)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
finish_reason_count += 1
|
||||
# finish reason should only return in last block
|
||||
assert finish_reason_count == 1
|
||||
assert chunk.choices[0].finish_reason == stop_reason
|
||||
assert delta.content
|
||||
assert "".join(chunks) == output
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize(
|
||||
"video_urls", [TEST_VIDEO_URLS[:i] for i in range(2, len(TEST_VIDEO_URLS))]
|
||||
)
|
||||
@pytest.mark.flaky(
|
||||
reruns=2,
|
||||
reruns_delay=5,
|
||||
condition=current_platform.is_rocm(),
|
||||
)
|
||||
async def test_multi_video_input(
|
||||
client: openai.AsyncOpenAI, model_name: str, video_urls: list[str]
|
||||
):
|
||||
messages = dummy_messages_from_video_url(video_urls)
|
||||
|
||||
if len(video_urls) > MAXIMUM_VIDEOS:
|
||||
with pytest.raises(openai.BadRequestError): # test multi-video input
|
||||
await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
)
|
||||
|
||||
# the server should still work afterwards
|
||||
completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=[0, 0, 0, 0, 0],
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
completion = completion.choices[0].text
|
||||
assert completion is not None and len(completion) >= 0
|
||||
else:
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
)
|
||||
message = chat_completion.choices[0].message
|
||||
assert message.content is not None and len(message.content) >= 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_omitted_video_url_requires_cached_uuid(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
):
|
||||
video_uuid = "test-video-uuid"
|
||||
video_url = TEST_VIDEO_URLS[0]
|
||||
|
||||
omitted_video_messages = describe_video_messages(
|
||||
None,
|
||||
extra_video_fields={"uuid": video_uuid},
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
openai.BadRequestError,
|
||||
match="Cache miss for video at index 0 but data is not provided",
|
||||
):
|
||||
await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=omitted_video_messages,
|
||||
)
|
||||
|
||||
await complete_and_check(
|
||||
client,
|
||||
model_name,
|
||||
describe_video_messages(
|
||||
video_url,
|
||||
extra_video_fields={"uuid": video_uuid},
|
||||
),
|
||||
context="populate UUID cache",
|
||||
)
|
||||
|
||||
await complete_and_check(
|
||||
client,
|
||||
model_name,
|
||||
omitted_video_messages,
|
||||
context="reuse cached video without URL",
|
||||
)
|
||||
@@ -1,676 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from transformers import AutoProcessor
|
||||
|
||||
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
||||
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
from vllm.multimodal.media import MediaWithBytes
|
||||
from vllm.multimodal.utils import encode_image_url, fetch_image
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL_NAME = "microsoft/Phi-3.5-vision-instruct"
|
||||
MAXIMUM_IMAGES = 2
|
||||
|
||||
# Required terms for beam search validation
|
||||
# Each entry is a list of term groups - ALL groups must match
|
||||
# Each group is a list of alternatives - at least ONE term in the group must appear
|
||||
# This provides semantic validation while allowing wording variation
|
||||
REQUIRED_BEAM_SEARCH_TERMS = [
|
||||
# Boardwalk image: must have "boardwalk" AND ("wooden" or "wood")
|
||||
[["boardwalk"], ["wooden", "wood"]],
|
||||
# Parrots image: must have ("parrot" or "bird") AND "two"
|
||||
[["parrot", "bird"], ["two"]],
|
||||
# Venn diagram: must have "venn" AND "diagram"
|
||||
[["venn"], ["diagram"]],
|
||||
# Gradient image: must have "gradient" AND ("color" or "spectrum")
|
||||
[["gradient"], ["color", "spectrum"]],
|
||||
]
|
||||
|
||||
|
||||
def check_output_matches_terms(content: str, term_groups: list[list[str]]) -> bool:
|
||||
"""Check if content matches all required term groups.
|
||||
Each term group requires at least one of its terms to be present.
|
||||
All term groups must be satisfied.
|
||||
"""
|
||||
content_lower = content.lower()
|
||||
return all(
|
||||
any(term.lower() in content_lower for term in group) for group in term_groups
|
||||
)
|
||||
|
||||
|
||||
def assert_non_empty_content(chat_completion, *, context: str = "") -> str:
|
||||
"""Assert the first choice has non-empty string content; return it.
|
||||
|
||||
Provides a detailed failure message including the full ChatCompletion
|
||||
response so flaky / model-quality issues are easy to diagnose.
|
||||
"""
|
||||
prefix = f"[{context}] " if context else ""
|
||||
choice = chat_completion.choices[0]
|
||||
content = choice.message.content
|
||||
|
||||
assert content is not None, (
|
||||
f"{prefix}Expected non-None content but got None. "
|
||||
f"finish_reason={choice.finish_reason!r}, "
|
||||
f"full message={choice.message!r}, "
|
||||
f"usage={chat_completion.usage!r}"
|
||||
)
|
||||
assert isinstance(content, str), (
|
||||
f"{prefix}Expected str content, got {type(content).__name__}: {content!r}"
|
||||
)
|
||||
assert len(content) > 0, (
|
||||
f"{prefix}Expected non-empty content but got empty string. "
|
||||
f"finish_reason={choice.finish_reason!r}, "
|
||||
f"full message={choice.message!r}, "
|
||||
f"usage={chat_completion.usage!r}"
|
||||
)
|
||||
return content
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"generate",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"5",
|
||||
"--enforce-eager",
|
||||
"--trust-remote-code",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"image": MAXIMUM_IMAGES}),
|
||||
*ROCM_EXTRA_ARGS,
|
||||
]
|
||||
|
||||
# ROCm: Allow more time to download videos from external sources.
|
||||
env_overrides = {
|
||||
**(
|
||||
{
|
||||
"VLLM_VIDEO_FETCH_TIMEOUT": "120",
|
||||
}
|
||||
if current_platform.is_rocm()
|
||||
else {}
|
||||
),
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_overrides) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def url_encoded_image(local_asset_server) -> dict[str, str]:
|
||||
return {
|
||||
image_asset: encode_image_url(local_asset_server.get_image_asset(image_asset))
|
||||
for image_asset in TEST_IMAGE_ASSETS
|
||||
}
|
||||
|
||||
|
||||
def dummy_messages_from_image_url(
|
||||
image_urls: str | list[str],
|
||||
content_text: str = "What's in this image?",
|
||||
):
|
||||
if isinstance(image_urls, str):
|
||||
image_urls = [image_urls]
|
||||
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*(
|
||||
{"type": "image_url", "image_url": {"url": image_url}}
|
||||
for image_url in image_urls
|
||||
),
|
||||
{"type": "text", "text": content_text},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def describe_image_messages(
|
||||
image_url: str, *, extra_image_fields: dict | None = None
|
||||
) -> list[dict]:
|
||||
"""Build the system + user messages used by the completions-with-image
|
||||
family of tests. *extra_image_fields* is merged into the top-level
|
||||
image content block (for uuid / bad-key tests)."""
|
||||
image_block: dict = {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
}
|
||||
if extra_image_fields:
|
||||
image_block.update(extra_image_fields)
|
||||
|
||||
return [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this image."},
|
||||
image_block,
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
async def complete_and_check(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
messages: list[dict],
|
||||
*,
|
||||
context: str,
|
||||
max_completion_tokens: int = 50,
|
||||
temperature: float = 0.0,
|
||||
) -> str:
|
||||
"""Run a chat completion and assert the output is non-empty.
|
||||
Returns the content string."""
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
temperature=temperature,
|
||||
)
|
||||
return assert_non_empty_content(chat_completion, context=context)
|
||||
|
||||
|
||||
def get_hf_prompt_tokens(model_name, content, image_url):
|
||||
processor = AutoProcessor.from_pretrained(
|
||||
model_name, trust_remote_code=True, num_crops=4
|
||||
)
|
||||
|
||||
placeholder = "<|image_1|>\n"
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"{placeholder}{content}",
|
||||
}
|
||||
]
|
||||
image = fetch_image(image_url)
|
||||
# Unwrap MediaWithBytes if present
|
||||
if isinstance(image, MediaWithBytes):
|
||||
image = image.media
|
||||
images = [image]
|
||||
|
||||
prompt = processor.tokenizer.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
inputs = processor(prompt, images, return_tensors="pt")
|
||||
|
||||
return inputs.input_ids.shape[1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
|
||||
async def test_single_chat_session_image(
|
||||
client: openai.AsyncOpenAI, model_name: str, image_url: str
|
||||
):
|
||||
content_text = "What's in this image?"
|
||||
messages = dummy_messages_from_image_url(image_url, content_text)
|
||||
|
||||
max_completion_tokens = 10
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
logprobs=True,
|
||||
temperature=0.0,
|
||||
top_logprobs=5,
|
||||
)
|
||||
assert len(chat_completion.choices) == 1, (
|
||||
f"Expected 1 choice, got {len(chat_completion.choices)}"
|
||||
)
|
||||
|
||||
choice = chat_completion.choices[0]
|
||||
assert choice.finish_reason == "length", (
|
||||
f"Expected finish_reason='length' (capped at {max_completion_tokens} "
|
||||
f"tokens), got {choice.finish_reason!r}. "
|
||||
f"content={choice.message.content!r}"
|
||||
)
|
||||
|
||||
hf_prompt_tokens = get_hf_prompt_tokens(model_name, content_text, image_url)
|
||||
expected_usage = openai.types.CompletionUsage(
|
||||
completion_tokens=max_completion_tokens,
|
||||
prompt_tokens=hf_prompt_tokens,
|
||||
total_tokens=hf_prompt_tokens + max_completion_tokens,
|
||||
)
|
||||
assert chat_completion.usage == expected_usage, (
|
||||
f"Usage mismatch: got {chat_completion.usage!r}, expected {expected_usage!r}"
|
||||
)
|
||||
|
||||
message = choice.message
|
||||
assert message.content is not None and len(message.content) >= 10, (
|
||||
f"Expected content with >=10 chars, got {message.content!r}"
|
||||
)
|
||||
assert message.role == "assistant", (
|
||||
f"Expected role='assistant', got {message.role!r}"
|
||||
)
|
||||
|
||||
messages.append({"role": "assistant", "content": message.content})
|
||||
|
||||
# test multi-turn dialogue
|
||||
messages.append({"role": "user", "content": "express your result in json"})
|
||||
await complete_and_check(
|
||||
client,
|
||||
model_name,
|
||||
messages,
|
||||
context=f"multi-turn follow-up for {image_url}",
|
||||
max_completion_tokens=10,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
|
||||
async def test_error_on_invalid_image_url_type(
|
||||
client: openai.AsyncOpenAI, model_name: str, image_url: str
|
||||
):
|
||||
content_text = "What's in this image?"
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": image_url},
|
||||
{"type": "text", "text": content_text},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
# image_url should be a dict {"url": "some url"}, not directly a string
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
|
||||
async def test_single_chat_session_image_beamsearch(
|
||||
client: openai.AsyncOpenAI, model_name: str, image_url: str
|
||||
):
|
||||
content_text = "What's in this image?"
|
||||
messages = dummy_messages_from_image_url(image_url, content_text)
|
||||
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
n=2,
|
||||
max_completion_tokens=10,
|
||||
logprobs=True,
|
||||
top_logprobs=5,
|
||||
extra_body=dict(use_beam_search=True),
|
||||
)
|
||||
assert len(chat_completion.choices) == 2, (
|
||||
f"Expected 2 beam search choices, got {len(chat_completion.choices)}"
|
||||
)
|
||||
|
||||
content_0 = chat_completion.choices[0].message.content
|
||||
content_1 = chat_completion.choices[1].message.content
|
||||
assert content_0 != content_1, (
|
||||
f"Beam search should produce different outputs for {image_url}, "
|
||||
f"but both returned: {content_0!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("raw_image_url", TEST_IMAGE_ASSETS)
|
||||
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
|
||||
async def test_single_chat_session_image_base64encoded(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
raw_image_url: str,
|
||||
image_url: str,
|
||||
url_encoded_image: dict[str, str],
|
||||
):
|
||||
content_text = "What's in this image?"
|
||||
messages = dummy_messages_from_image_url(
|
||||
url_encoded_image[raw_image_url],
|
||||
content_text,
|
||||
)
|
||||
|
||||
max_completion_tokens = 10
|
||||
# test single completion
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=max_completion_tokens,
|
||||
logprobs=True,
|
||||
temperature=0.0,
|
||||
top_logprobs=5,
|
||||
)
|
||||
assert len(chat_completion.choices) == 1, (
|
||||
f"Expected 1 choice, got {len(chat_completion.choices)}"
|
||||
)
|
||||
|
||||
choice = chat_completion.choices[0]
|
||||
assert choice.finish_reason == "length", (
|
||||
f"Expected finish_reason='length', got {choice.finish_reason!r}. "
|
||||
f"content={choice.message.content!r}"
|
||||
)
|
||||
|
||||
hf_prompt_tokens = get_hf_prompt_tokens(model_name, content_text, image_url)
|
||||
expected_usage = openai.types.CompletionUsage(
|
||||
completion_tokens=max_completion_tokens,
|
||||
prompt_tokens=hf_prompt_tokens,
|
||||
total_tokens=hf_prompt_tokens + max_completion_tokens,
|
||||
)
|
||||
assert chat_completion.usage == expected_usage, (
|
||||
f"Usage mismatch: got {chat_completion.usage!r}, expected {expected_usage!r}"
|
||||
)
|
||||
|
||||
message = choice.message
|
||||
assert message.content is not None and len(message.content) >= 10, (
|
||||
f"Expected content with >=10 chars, got {message.content!r}"
|
||||
)
|
||||
assert message.role == "assistant", (
|
||||
f"Expected role='assistant', got {message.role!r}"
|
||||
)
|
||||
|
||||
messages.append({"role": "assistant", "content": message.content})
|
||||
|
||||
# test multi-turn dialogue
|
||||
messages.append({"role": "user", "content": "express your result in json"})
|
||||
await complete_and_check(
|
||||
client,
|
||||
model_name,
|
||||
messages,
|
||||
context=f"multi-turn base64 follow-up for {raw_image_url}",
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("image_idx", list(range(len(TEST_IMAGE_ASSETS))))
|
||||
async def test_single_chat_session_image_base64encoded_beamsearch(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
image_idx: int,
|
||||
url_encoded_image: dict[str, str],
|
||||
):
|
||||
# NOTE: This test validates that we pass MM data through beam search
|
||||
raw_image_url = TEST_IMAGE_ASSETS[image_idx]
|
||||
required_terms = REQUIRED_BEAM_SEARCH_TERMS[image_idx]
|
||||
|
||||
messages = dummy_messages_from_image_url(url_encoded_image[raw_image_url])
|
||||
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
n=2,
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
extra_body=dict(use_beam_search=True),
|
||||
)
|
||||
assert len(chat_completion.choices) == 2, (
|
||||
f"Expected 2 beam search choices for image {image_idx} "
|
||||
f"({raw_image_url}), got {len(chat_completion.choices)}"
|
||||
)
|
||||
|
||||
# Verify beam search produces two different non-empty outputs
|
||||
content_0 = chat_completion.choices[0].message.content
|
||||
content_1 = chat_completion.choices[1].message.content
|
||||
|
||||
# Emit beam search outputs for debugging
|
||||
print(
|
||||
f"Beam search outputs for image {image_idx} ({raw_image_url}): "
|
||||
f"Output 0: {content_0!r}, Output 1: {content_1!r}"
|
||||
)
|
||||
|
||||
assert content_0, (
|
||||
f"First beam output is empty for image {image_idx} ({raw_image_url}). "
|
||||
f"finish_reason={chat_completion.choices[0].finish_reason!r}"
|
||||
)
|
||||
assert content_1, (
|
||||
f"Second beam output is empty for image {image_idx} "
|
||||
f"({raw_image_url}). "
|
||||
f"finish_reason={chat_completion.choices[1].finish_reason!r}"
|
||||
)
|
||||
assert content_0 != content_1, (
|
||||
f"Beam search produced identical outputs for image {image_idx} "
|
||||
f"({raw_image_url}): {content_0!r}"
|
||||
)
|
||||
|
||||
# Verify each output contains the required terms for this image
|
||||
for i, content in enumerate([content_0, content_1]):
|
||||
assert check_output_matches_terms(content, required_terms), (
|
||||
f"Beam output {i} for image {image_idx} ({raw_image_url}) "
|
||||
f"doesn't match required terms.\n"
|
||||
f" content: {content!r}\n"
|
||||
f" required (all groups, >=1 per group): {required_terms}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
|
||||
async def test_chat_streaming_image(
|
||||
client: openai.AsyncOpenAI, model_name: str, image_url: str
|
||||
):
|
||||
messages = dummy_messages_from_image_url(image_url)
|
||||
|
||||
# test single completion
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
)
|
||||
output = chat_completion.choices[0].message.content
|
||||
stop_reason = chat_completion.choices[0].finish_reason
|
||||
|
||||
# test streaming
|
||||
stream = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
)
|
||||
chunks: list[str] = []
|
||||
finish_reason_count = 0
|
||||
async for chunk in stream:
|
||||
delta = chunk.choices[0].delta
|
||||
if delta.role:
|
||||
assert delta.role == "assistant", (
|
||||
f"Expected role='assistant' in stream delta, got {delta.role!r}"
|
||||
)
|
||||
if delta.content:
|
||||
chunks.append(delta.content)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
finish_reason_count += 1
|
||||
# finish reason should only return in last block
|
||||
assert finish_reason_count == 1, (
|
||||
f"Expected exactly 1 finish_reason across stream chunks, "
|
||||
f"got {finish_reason_count}"
|
||||
)
|
||||
assert chunk.choices[0].finish_reason == stop_reason, (
|
||||
f"Stream finish_reason={chunk.choices[0].finish_reason!r} "
|
||||
f"doesn't match non-stream finish_reason={stop_reason!r}"
|
||||
)
|
||||
|
||||
streamed_text = "".join(chunks)
|
||||
assert streamed_text == output, (
|
||||
f"Streamed output doesn't match non-streamed for {image_url}.\n"
|
||||
f" streamed: {streamed_text!r}\n"
|
||||
f" non-streamed: {output!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize(
|
||||
"image_urls",
|
||||
[TEST_IMAGE_ASSETS[:i] for i in range(2, len(TEST_IMAGE_ASSETS))],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_multi_image_input(
|
||||
client: openai.AsyncOpenAI, model_name: str, image_urls: list[str]
|
||||
):
|
||||
messages = dummy_messages_from_image_url(image_urls)
|
||||
|
||||
if len(image_urls) > MAXIMUM_IMAGES:
|
||||
with pytest.raises(openai.BadRequestError): # test multi-image input
|
||||
await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
)
|
||||
|
||||
# the server should still work afterwards
|
||||
completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=[0, 0, 0, 0, 0],
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
assert completion.choices[0].text is not None, (
|
||||
"Server failed to produce output after rejecting over-limit "
|
||||
"multi-image request"
|
||||
)
|
||||
else:
|
||||
await complete_and_check(
|
||||
client,
|
||||
model_name,
|
||||
messages,
|
||||
context=f"multi-image input ({len(image_urls)} images)",
|
||||
max_completion_tokens=10,
|
||||
temperature=0.0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize(
|
||||
"image_urls",
|
||||
[TEST_IMAGE_ASSETS[:i] for i in range(2, len(TEST_IMAGE_ASSETS))],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_completions_with_image(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
image_urls: list[str],
|
||||
):
|
||||
for image_url in image_urls:
|
||||
messages = describe_image_messages(image_url)
|
||||
await complete_and_check(
|
||||
client,
|
||||
model_name,
|
||||
messages,
|
||||
context=f"completions_with_image url={image_url}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize(
|
||||
"image_urls",
|
||||
[TEST_IMAGE_ASSETS[:i] for i in range(2, len(TEST_IMAGE_ASSETS))],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_completions_with_image_with_uuid(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
image_urls: list[str],
|
||||
):
|
||||
for image_url in image_urls:
|
||||
messages = describe_image_messages(
|
||||
image_url,
|
||||
extra_image_fields={"uuid": image_url},
|
||||
)
|
||||
await complete_and_check(
|
||||
client,
|
||||
model_name,
|
||||
messages,
|
||||
context=f"uuid first request url={image_url}",
|
||||
)
|
||||
|
||||
cached_messages: list[dict] = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this image."},
|
||||
{"type": "image_url", "image_url": {}, "uuid": image_url},
|
||||
],
|
||||
},
|
||||
]
|
||||
await complete_and_check(
|
||||
client,
|
||||
model_name,
|
||||
cached_messages,
|
||||
context=f"uuid cached (empty image) uuid={image_url}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_completions_with_empty_image_with_uuid_without_cache_hit(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
):
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await client.chat.completions.create(
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this image."},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {},
|
||||
"uuid": "uuid_not_previously_seen",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
model=model_name,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize(
|
||||
"image_urls",
|
||||
[TEST_IMAGE_ASSETS[:i] for i in range(2, len(TEST_IMAGE_ASSETS))],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_completions_with_image_with_incorrect_uuid_format(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
image_urls: list[str],
|
||||
):
|
||||
for image_url in image_urls:
|
||||
messages = describe_image_messages(
|
||||
image_url,
|
||||
extra_image_fields={
|
||||
"also_incorrect_uuid_key": image_url,
|
||||
},
|
||||
)
|
||||
# Inject the bad key inside image_url dict too
|
||||
messages[1]["content"][1]["image_url"]["incorrect_uuid_key"] = image_url
|
||||
|
||||
await complete_and_check(
|
||||
client,
|
||||
model_name,
|
||||
messages,
|
||||
context=f"incorrect uuid format url={image_url}",
|
||||
)
|
||||
Reference in New Issue
Block a user