[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,192 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test that ``InputProcessor.inject_into_mm_cache()`` correctly injects
pre-processed mm_kwargs into the processor cache and reports MM cache
hit rate metrics accurately.
This is used by frameworks like Dynamo that run the HF processor on a
frontend and transfer pre-processed mm_kwargs to the backend, avoiding
redundant processing.
"""
import logging
import pytest
import regex as re
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
from vllm import LLM, SamplingParams
from vllm.renderers.params import ChatParams
from vllm.v1.metrics import loggers as stat_loggers
from vllm.v1.metrics.reader import Counter, Metric
def _make_messages(image_url: str):
return [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {"url": image_url},
},
],
}
]
def _get_counter_value(metrics: list[Metric], name: str):
metric = next(m for m in metrics if m.name == name)
assert isinstance(metric, Counter)
return metric.value
def _get_mm_cache_stats(metrics: list[Metric]):
mm_cache_queries = _get_counter_value(metrics, "vllm:mm_cache_queries")
mm_cache_hits = _get_counter_value(metrics, "vllm:mm_cache_hits")
return mm_cache_queries, mm_cache_hits
def _get_mm_cache_log(llm: LLM, caplog_vllm: pytest.LogCaptureFixture) -> float:
caplog_vllm.clear()
with caplog_vllm.at_level(logging.INFO, logger=stat_loggers.__name__):
llm.llm_engine.do_log_stats()
assert len(caplog_vllm.records) == 1
msg = caplog_vllm.records[0].getMessage()
assert "MM cache hit rate" in msg
match = re.search(r"MM cache hit rate: ([0-9.]+)%", msg)
assert match is not None
return float(match.group(1))
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:2]], indirect=True)
@pytest.mark.parametrize("mm_processor_cache_type", ["lru", "shm"])
def test_inject_into_mm_cache(
num_gpus_available,
image_urls,
mm_processor_cache_type,
caplog_vllm,
vllm_runner,
):
"""Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
the processor cache and MM cache hit metrics are updated correctly.
Steps:
1. Two normal requests (same image) -> cache miss then hit (baseline)
2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
then generate with a pre-rendered input -> verifies injection works
"""
with vllm_runner(
"llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
mm_processor_cache_type=mm_processor_cache_type,
) as runner:
# Step 1: Normal requests to populate the cache
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(50.0)
# Step 2: Use a second image to get valid expanded tokens and
# placeholder positions via the renderer.
runner.llm.chat(_make_messages(image_urls[1]))
queries_before = _get_mm_cache_stats(runner.llm.get_metrics())[0] # 3
renderer = runner.llm.llm_engine.renderer
cache = renderer.mm_processor_cache
assert cache is not None, "Processor cache should be enabled"
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[1])],
ChatParams(),
)
eng_input = eng_prompts[0]
# Inject pre-processed mm_kwargs with a NEW hash via public API
new_mm_hash = "deadbeef" * 8
mm_hashes = {"image": [new_mm_hash]}
mm_kwargs = eng_input["mm_kwargs"]
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# Build pre-rendered input (no externally_processed flag needed)
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
runner.llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
# Verify cache was queried and injection happened
queries_after = _get_mm_cache_stats(runner.llm.get_metrics())[0]
assert queries_after > queries_before, (
"Cache should have been queried for the injected item"
)
mm_rate = _get_mm_cache_log(runner.llm, caplog_vllm)
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True)
def test_inject_into_mm_cache_without_cache(
num_gpus_available,
image_urls,
vllm_runner,
):
"""Test that inject_into_mm_cache works gracefully when processor cache
is disabled (mm_processor_cache_gb=0). Should not crash.
"""
with vllm_runner(
"llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
mm_processor_cache_gb=0,
) as runner:
# Run a normal chat request first to warm up the model.
runner.llm.chat(_make_messages(image_urls[0]))
# Use the renderer to get a proper EngineInput with expanded tokens
renderer = runner.llm.llm_engine.renderer
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[0])],
ChatParams(),
)
eng_input = eng_prompts[0]
mm_hashes = {"image": ["abcd1234" * 8]}
mm_kwargs = eng_input["mm_kwargs"]
# inject_into_mm_cache should not crash even without cache
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# Build and generate with pre-rendered input
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
result = runner.llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
assert len(result) == 1, "Should produce one output"
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
@@ -1,97 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import pytest
import regex as re
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
from vllm import LLM
from vllm.renderers.chat_utils import ChatCompletionMessageParam
from vllm.v1.metrics import loggers as stat_loggers
from vllm.v1.metrics.reader import Counter, Metric
def _make_messages(image_url: str) -> list[ChatCompletionMessageParam]:
return [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {"url": image_url},
},
],
}
]
def _get_counter_value(metrics: list[Metric], name: str):
metric = next(m for m in metrics if m.name == name)
assert isinstance(metric, Counter)
return metric.value
def _get_mm_cache_stats(metrics: list[Metric]):
mm_cache_queries = _get_counter_value(metrics, "vllm:mm_cache_queries")
mm_cache_hits = _get_counter_value(metrics, "vllm:mm_cache_hits")
return mm_cache_queries, mm_cache_hits
def _get_mm_cache_log(llm: LLM, caplog_vllm: pytest.LogCaptureFixture) -> float:
caplog_vllm.clear()
with caplog_vllm.at_level(logging.INFO, logger=stat_loggers.__name__):
llm.llm_engine.do_log_stats()
assert len(caplog_vllm.records) == 1
msg = caplog_vllm.records[0].getMessage()
assert "MM cache hit rate" in msg
match = re.search(r"MM cache hit rate: ([0-9.]+)%", msg)
assert match is not None
return float(match.group(1))
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:2]], indirect=True)
@pytest.mark.parametrize("mm_processor_cache_type", ["lru", "shm"])
def test_mm_cache_stats(
num_gpus_available,
image_urls,
mm_processor_cache_type,
caplog_vllm,
vllm_runner,
):
with vllm_runner(
"llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
mm_processor_cache_type=mm_processor_cache_type,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
) as runner:
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
runner.llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 0)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (3, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(33.3)
# NOTE: This only resets hit rate stats in CachingMetrics
# The raw queries and hits counts remain unaffected
runner.llm.reset_mm_cache()
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (4, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
runner.llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (5, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
@@ -1,64 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
from vllm.exceptions import VLLMValidationError
MODEL = "llava-hf/llava-1.5-7b-hf"
PROMPT = "USER: <image>\nDescribe this image briefly.\nASSISTANT:"
TEXT_ONLY_PROMPT = "USER: What is 2 + 2?\nASSISTANT:"
@pytest.fixture(scope="module")
def llm(vllm_runner):
"""LLM with enable_mm_embeds=True and all modality limits zeroed out."""
with vllm_runner(
MODEL,
max_model_len=2048,
enforce_eager=True,
gpu_memory_utilization=0.8,
enable_mm_embeds=True,
limit_mm_per_prompt={"image": 0},
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
def test_generate_with_embedding(llm: LLM):
"""Pre-computed embedding produces tokens without hanging."""
embedding = ImageAsset("stop_sign").image_embeds
outputs = llm.generate(
{"prompt": PROMPT, "multi_modal_data": {"image": embedding}},
sampling_params=SamplingParams(max_tokens=32, temperature=0.0),
)
assert len(outputs) == 1
assert len(outputs[0].outputs[0].text) > 0
@pytest.mark.skip_global_cleanup
def test_raw_image_rejected(llm: LLM):
"""Raw image input is still rejected when limit=0."""
raw_image = ImageAsset("stop_sign").pil_image
with pytest.raises(VLLMValidationError, match=r"At most 0 image\(s\)"):
llm.generate(
{"prompt": PROMPT, "multi_modal_data": {"image": raw_image}},
sampling_params=SamplingParams(max_tokens=16),
)
@pytest.mark.skip_global_cleanup
def test_text_only_prompt(llm: LLM):
"""Text-only prompts still work under this config."""
outputs = llm.generate(
TEXT_ONLY_PROMPT,
sampling_params=SamplingParams(max_tokens=16, temperature=0.0),
)
assert len(outputs) == 1
assert len(outputs[0].outputs[0].text) > 0
@@ -1,447 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from vllm import LLM, SamplingParams
from vllm.inputs import TextPrompt
from vllm.outputs import RequestOutput
from vllm.renderers.chat_utils import ChatCompletionMessageParam
def _make_mock_llm() -> LLM:
llm = object.__new__(LLM)
llm.chat_template = None
llm.model_config = SimpleNamespace(
runner_type="generate", enable_prompt_embeds=False
)
return llm
def test_generate_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 4}
sampling_params = SamplingParams(max_tokens=1)
_run_completion = Mock(return_value=["ok"])
monkeypatch.setattr(llm, "_run_completion", _run_completion)
outputs = llm.generate(
"prompt",
sampling_params=sampling_params,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == ["ok"]
assert _run_completion.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_enqueue_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_resize": False}
sampling_params = SamplingParams(max_tokens=1)
_add_completion_requests = Mock(return_value=["req-0"])
monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests)
request_ids = llm.enqueue(
"prompt",
sampling_params=sampling_params,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert request_ids == ["req-0"]
assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_pan_and_scan": True}
sampling_params = SamplingParams(max_tokens=1)
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
_run_chat = Mock(return_value=["ok"])
monkeypatch.setattr(llm, "_run_chat", _run_chat)
outputs = llm.chat(
messages,
sampling_params=sampling_params,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == ["ok"]
assert _run_chat.call_args.kwargs["mm_processor_kwargs"] == mm_processor_kwargs
def test_enqueue_chat_forwards_mm_processor_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_pan_and_scan": True}
sampling_params = SamplingParams(max_tokens=1)
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
_add_chat_requests = Mock(return_value=["req-0"])
monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests)
request_ids = llm.enqueue_chat(
messages,
sampling_params=sampling_params,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert request_ids == ["req-0"]
assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_run_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
sampling_params = SamplingParams(max_tokens=1)
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
sentinel_output = ["done"]
_add_chat_requests = Mock()
monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests)
_run_engine = Mock(return_value=sentinel_output)
monkeypatch.setattr(llm, "_run_engine", _run_engine)
outputs = llm._run_chat(
messages=messages,
params=sampling_params,
output_type=RequestOutput,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == sentinel_output
assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_run_completion_forwards_mm_processor_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"min_pixels": 4 * 28 * 28}
sampling_params = SamplingParams(max_tokens=1)
sentinel_output = ["done"]
_add_completion_requests = Mock()
monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests)
_run_engine = Mock(return_value=sentinel_output)
monkeypatch.setattr(llm, "_run_engine", _run_engine)
outputs = llm._run_completion(
prompts=["prompt"],
params=sampling_params,
output_type=RequestOutput,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == sentinel_output
assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_add_completion_requests_forwards_mm_processor_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"max_dynamic_patch": 4}
sampling_params = SamplingParams(max_tokens=1)
_params_to_seq = Mock(return_value=[sampling_params])
monkeypatch.setattr(llm, "_params_to_seq", _params_to_seq)
_lora_request_to_seq = Mock(return_value=[None])
monkeypatch.setattr(llm, "_lora_request_to_seq", _lora_request_to_seq)
_priority_to_seq = Mock(return_value=[0])
monkeypatch.setattr(llm, "_priority_to_seq", _priority_to_seq)
_preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
monkeypatch.setattr(llm, "_preprocess_cmpl_one", _preprocess_cmpl_one)
captured_prompts = []
def fake_render_and_add_requests(*, prompts, **_kwargs):
captured_prompts.extend(prompts)
return ["req-0"]
_render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
monkeypatch.setattr(llm, "_render_and_add_requests", _render_and_add_requests)
request_ids = llm._add_completion_requests(
prompts=["prompt"],
params=sampling_params,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert request_ids == ["req-0"]
_preprocess_cmpl_one.assert_called_once_with(
"prompt",
None,
mm_processor_kwargs=mm_processor_kwargs,
)
assert captured_prompts == [{"prompt_token_ids": [1]}]
def test_preprocess_cmpl_applies_mm_processor_kwargs_to_renderer(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
prompt: TextPrompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
renderer = Mock()
renderer.default_cmpl_tok_params = Mock()
renderer.default_cmpl_tok_params.with_kwargs.return_value = "tok-params"
renderer.render_cmpl.return_value = ["engine-input"]
llm.renderer = renderer
monkeypatch.setattr(
"vllm.entrypoints.common.offline.parse_model_prompt",
lambda _model_config, parsed_prompt: parsed_prompt,
)
outputs = llm._preprocess_cmpl(
[prompt],
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == ["engine-input"]
renderer.render_cmpl.assert_called_once_with(
[prompt],
"tok-params",
prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
)
def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
prompt: TextPrompt = {
"prompt": "<image>",
"multi_modal_data": {"image": object()},
"mm_processor_kwargs": {"num_crops": 2},
}
renderer = Mock()
renderer.default_cmpl_tok_params = Mock()
renderer.default_cmpl_tok_params.with_kwargs.return_value = "tok-params"
renderer.render_cmpl.return_value = ["engine-input"]
llm.renderer = renderer
monkeypatch.setattr(
"vllm.entrypoints.common.offline.parse_model_prompt",
lambda _model_config, parsed_prompt: parsed_prompt,
)
outputs = llm._preprocess_cmpl([prompt])
assert outputs == ["engine-input"]
renderer.render_cmpl.assert_called_once_with(
[prompt],
"tok-params",
prompt_extras=None,
)
def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
messages: list[list[ChatCompletionMessageParam]] = [
[{"role": "user", "content": "Describe this image."}]
]
renderer = Mock()
renderer.tokenizer = object()
renderer.default_chat_tok_params = Mock()
renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
renderer.render_chat.return_value = (messages, ["engine-input"])
llm.renderer = renderer
outputs = llm._preprocess_chat(
messages,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == ["engine-input"]
call_args = renderer.render_chat.call_args
assert call_args.args[0] == messages
assert call_args.args[1].mm_processor_kwargs == mm_processor_kwargs
assert call_args.args[2] == "tok-params"
assert call_args.kwargs["prompt_extras"] == {
"mm_processor_kwargs": mm_processor_kwargs
}
def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
llm = _make_mock_llm()
messages: list[list[ChatCompletionMessageParam]] = [
[{"role": "user", "content": "Describe this image."}]
]
renderer = Mock()
renderer.tokenizer = object()
renderer.default_chat_tok_params = Mock()
renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
renderer.render_chat.return_value = (messages, ["engine-input"])
llm.renderer = renderer
outputs = llm._preprocess_chat(messages)
assert outputs == ["engine-input"]
call_args = renderer.render_chat.call_args
assert call_args.args[0] == messages
assert call_args.args[1].mm_processor_kwargs is None
assert call_args.args[2] == "tok-params"
assert call_args.kwargs["prompt_extras"] is None
def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None:
# Matches `ChatCompletionRequest.add_special_tokens` on the server.
llm = _make_mock_llm()
messages: list[list[ChatCompletionMessageParam]] = [
[{"role": "user", "content": "hi"}]
]
renderer = Mock()
renderer.tokenizer = object()
renderer.default_chat_tok_params = Mock()
renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
renderer.render_chat.return_value = (messages, ["engine-input"])
llm.renderer = renderer
llm._preprocess_chat(messages)
renderer.default_chat_tok_params.with_kwargs.assert_called_once_with(
add_special_tokens=False
)
def test_preprocess_chat_tokenization_kwargs_override_add_special_tokens() -> None:
llm = _make_mock_llm()
messages: list[list[ChatCompletionMessageParam]] = [
[{"role": "user", "content": "hi"}]
]
renderer = Mock()
renderer.tokenizer = object()
renderer.default_chat_tok_params = Mock()
renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
renderer.render_chat.return_value = (messages, ["engine-input"])
llm.renderer = renderer
llm._preprocess_chat(
messages,
tokenization_kwargs={"add_special_tokens": True, "truncate_prompt_tokens": 8},
)
renderer.default_chat_tok_params.with_kwargs.assert_called_once_with(
add_special_tokens=True, truncate_prompt_tokens=8
)
@pytest.fixture(scope="module")
def llava_llm():
from vllm.config import ModelConfig, VllmConfig
from vllm.renderers.hf import HfRenderer
from vllm.tokenizers import cached_tokenizer_from_config
# A real multimodal model (Llama tokenizer, BOS=1) so that the real
# processor default (`add_special_tokens=True`) is in play. Only
# config/tokenizer/processor files are fetched, no weights.
model_config = ModelConfig(model="llava-hf/llava-1.5-7b-hf", max_model_len=128)
renderer = HfRenderer(
VllmConfig(model_config=model_config),
cached_tokenizer_from_config(model_config),
)
assert renderer.default_chat_tok_params.add_special_tokens is True
llm = _make_mock_llm()
# `_preprocess_cmpl` parses prompts against the real model config.
llm.model_config = model_config
llm.renderer = renderer
return llm
class TestChatAddSpecialTokensDefault:
"""`LLM.chat()` renders the chat template to text and then tokenizes it.
The multimodal processor default is `add_special_tokens=True`, so a
template that emits `bos_token` used to get a second BOS from the
tokenizer. `_preprocess_chat` now defaults `add_special_tokens=False`
like the online chat API does (`ChatCompletionRequest`), unless the
caller overrides it via `tokenization_kwargs`.
"""
# Mirrors the Gemma 3 chat template and the bundled deepseek_vl2 /
# deepseek_ocr templates, whose rendered output starts with the BOS token.
BOS_TEMPLATE = (
"{{ bos_token }}{% for m in messages %}{{ m['content'] }}{% endfor %}"
)
MESSAGES = [[{"role": "user", "content": "hi"}]]
def _chat_token_ids(self, llm, **kwargs):
(engine_input,) = llm._preprocess_chat(
self.MESSAGES,
chat_template=self.BOS_TEMPLATE,
add_generation_prompt=False,
**kwargs,
)
return engine_input["prompt_token_ids"]
def test_chat_does_not_duplicate_template_bos(self, llava_llm):
bos = llava_llm.renderer.tokenizer.bos_token_id
prompt_token_ids = self._chat_token_ids(llava_llm)
assert prompt_token_ids[0] == bos
assert prompt_token_ids.count(bos) == 1
def test_chat_matches_online_chat_api(self, llava_llm):
from vllm.renderers.params import ChatParams, TokenizeParams
# `ChatCompletionRequest.add_special_tokens` defaults to `False`.
online_tok_params = TokenizeParams(
max_total_tokens=llava_llm.renderer.model_config.max_model_len,
add_special_tokens=False,
)
chat_params = ChatParams(
chat_template=self.BOS_TEMPLATE,
chat_template_kwargs=dict(tokenize=False, add_generation_prompt=False),
)
_, (online_input,) = llava_llm.renderer.render_chat(
self.MESSAGES, chat_params, online_tok_params
)
assert self._chat_token_ids(llava_llm) == online_input["prompt_token_ids"]
def test_explicit_tokenization_kwargs_override_default(self, llava_llm):
bos = llava_llm.renderer.tokenizer.bos_token_id
prompt_token_ids = self._chat_token_ids(
llava_llm, tokenization_kwargs={"add_special_tokens": True}
)
assert prompt_token_ids.count(bos) == 2
def test_generate_keeps_processor_default(self, llava_llm):
# Raw prompts have no chat template to emit BOS, so `LLM.generate()`
# must keep letting the tokenizer add it.
bos = llava_llm.renderer.tokenizer.bos_token_id
(engine_input,) = llava_llm._preprocess_cmpl(["hi"])
assert engine_input["prompt_token_ids"][0] == bos
@@ -1,515 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import openai
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
from vllm.assets.base import VLLM_S3_BUCKET_URL
from vllm.multimodal.utils import encode_video_url, fetch_video
from vllm.platforms import current_platform
MODEL_NAME = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
MAXIMUM_VIDEOS = 3
TEST_VIDEO_URLS = [
f"{VLLM_S3_BUCKET_URL}/multimodal_asset/slow_traffic_small.mp4",
f"{VLLM_S3_BUCKET_URL}/multimodal_asset/vtest.avi",
f"{VLLM_S3_BUCKET_URL}/multimodal_asset/Megamind.avi",
]
@pytest.fixture(scope="module")
def server():
args = [
"--runner",
"generate",
"--max-model-len",
"32768",
"--max-num-seqs",
"2",
"--enforce-eager",
"--trust-remote-code",
"--limit-mm-per-prompt",
json.dumps({"video": MAXIMUM_VIDEOS}),
"--media-io-kwargs",
json.dumps({"video": {"num_frames": 32}}),
"--trust-request-mm-kwargs",
]
# ROCm: Allow more time to download videos from external sources.
env_overrides = {}
if current_platform.is_rocm():
env_overrides = {
"VLLM_VIDEO_FETCH_TIMEOUT": "120",
}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_overrides) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
@pytest.fixture(scope="session")
def url_encoded_video() -> dict[str, str]:
return {
video_url: encode_video_url(fetch_video(video_url)[0])
for video_url in TEST_VIDEO_URLS
}
def assert_non_empty_content(chat_completion, *, context: str = "") -> str:
"""Assert the first choice has non-empty string content; return it."""
prefix = f"[{context}] " if context else ""
choice = chat_completion.choices[0]
content = choice.message.content
assert content is not None, (
f"{prefix}Expected non-None content but got None. "
f"finish_reason={choice.finish_reason!r}, "
f"full message={choice.message!r}, "
f"usage={chat_completion.usage!r}"
)
assert isinstance(content, str), (
f"{prefix}Expected str content, got {type(content).__name__}: {content!r}"
)
assert len(content) > 0, (
f"{prefix}Expected non-empty content but got empty string. "
f"finish_reason={choice.finish_reason!r}, "
f"full message={choice.message!r}, "
f"usage={chat_completion.usage!r}"
)
return content
def describe_video_messages(
video_url: str | None, *, extra_video_fields: dict | None = None
) -> list[dict]:
"""Build the system + user messages used by the completions-with-video
family of tests. *extra_video_fields* is merged into the top-level
video content block (for uuid / bad-key tests).
Pass ``None`` for *video_url* to omit media (uuid-backed cache hits).
"""
video_block: dict = {
"type": "video_url",
"video_url": {} if video_url is None else {"url": video_url},
}
if extra_video_fields:
video_block.update(extra_video_fields)
return [
{"role": "system", "content": "You are a helpful assistant."},
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this video."},
video_block,
],
},
]
async def complete_and_check(
client: openai.AsyncOpenAI,
model_name: str,
messages: list[dict],
*,
context: str,
max_completion_tokens: int = 50,
temperature: float = 0.0,
) -> str:
"""Run a chat completion and assert the output is non-empty."""
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=max_completion_tokens,
temperature=temperature,
)
return assert_non_empty_content(chat_completion, context=context)
def dummy_messages_from_video_url(
video_urls: str | list[str],
content_text: str = "What's in this video?",
):
if isinstance(video_urls, str):
video_urls = [video_urls]
return [
{
"role": "user",
"content": [
*(
{"type": "video_url", "video_url": {"url": video_url}}
for video_url in video_urls
),
{"type": "text", "text": content_text},
],
}
]
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
async def test_single_chat_session_video(
client: openai.AsyncOpenAI, model_name: str, video_url: str
):
messages = dummy_messages_from_video_url(video_url)
# test single completion
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
logprobs=True,
temperature=0.0,
top_logprobs=5,
)
assert len(chat_completion.choices) == 1
choice = chat_completion.choices[0]
assert choice.finish_reason == "length"
assert chat_completion.usage == openai.types.CompletionUsage(
completion_tokens=10, prompt_tokens=6287, total_tokens=6297
)
message = choice.message
message = chat_completion.choices[0].message
assert message.content is not None and len(message.content) >= 10
assert message.role == "assistant"
messages.append({"role": "assistant", "content": message.content})
# test multi-turn dialogue
messages.append({"role": "user", "content": "express your result in json"})
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
)
message = chat_completion.choices[0].message
assert message.content is not None and len(message.content) >= 0
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("video_url", [TEST_VIDEO_URLS[0]])
async def test_request_media_io_kwargs_override_uses_fewer_video_frames(
client: openai.AsyncOpenAI, model_name: str, video_url: str
):
messages = dummy_messages_from_video_url(video_url)
default_resp = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=1,
temperature=0.0,
)
override_resp = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=1,
temperature=0.0,
extra_body={
"media_io_kwargs": {
"video": {
"num_frames": 4,
}
}
},
)
assert default_resp.usage is not None
assert override_resp.usage is not None
assert override_resp.usage.prompt_tokens < default_resp.usage.prompt_tokens
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("video_url", [TEST_VIDEO_URLS[0]])
async def test_invalid_num_frames_request_recoverable(
client: openai.AsyncOpenAI, model_name: str, video_url: str
):
messages = dummy_messages_from_video_url(video_url)
with pytest.raises((openai.BadRequestError, openai.APIStatusError)):
await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=1,
temperature=0.0,
extra_body={
"media_io_kwargs": {
"video": {
"num_frames": "invalid",
}
}
},
)
# Server should still handle subsequent requests after the failed one.
recovery_resp = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=1,
temperature=0.0,
)
recovery_msg = recovery_resp.choices[0].message
assert recovery_msg.content is not None and len(recovery_msg.content) >= 0
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
async def test_error_on_invalid_video_url_type(
client: openai.AsyncOpenAI, model_name: str, video_url: str
):
messages = [
{
"role": "user",
"content": [
{"type": "video_url", "video_url": video_url},
{"type": "text", "text": "What's in this video?"},
],
}
]
# video_url should be a dict {"url": "some url"}, not directly a string
with pytest.raises(openai.BadRequestError):
_ = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
temperature=0.0,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
async def test_single_chat_session_video_beamsearch(
client: openai.AsyncOpenAI, model_name: str, video_url: str
):
messages = dummy_messages_from_video_url(video_url)
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
n=2,
max_completion_tokens=10,
logprobs=True,
top_logprobs=5,
extra_body=dict(use_beam_search=True),
)
assert len(chat_completion.choices) == 2
assert (
chat_completion.choices[0].message.content
!= chat_completion.choices[1].message.content
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
async def test_single_chat_session_video_base64encoded(
client: openai.AsyncOpenAI,
model_name: str,
video_url: str,
url_encoded_video: dict[str, str],
):
messages = dummy_messages_from_video_url(url_encoded_video[video_url])
# test single completion
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
logprobs=True,
temperature=0.0,
top_logprobs=5,
)
assert len(chat_completion.choices) == 1
choice = chat_completion.choices[0]
assert choice.finish_reason == "length"
assert chat_completion.usage == openai.types.CompletionUsage(
completion_tokens=10, prompt_tokens=6287, total_tokens=6297
)
message = choice.message
message = chat_completion.choices[0].message
assert message.content is not None and len(message.content) >= 10
assert message.role == "assistant"
messages.append({"role": "assistant", "content": message.content})
# test multi-turn dialogue
messages.append({"role": "user", "content": "express your result in json"})
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
temperature=0.0,
)
message = chat_completion.choices[0].message
assert message.content is not None and len(message.content) >= 0
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
async def test_single_chat_session_video_base64encoded_beamsearch(
client: openai.AsyncOpenAI,
model_name: str,
video_url: str,
url_encoded_video: dict[str, str],
):
messages = dummy_messages_from_video_url(url_encoded_video[video_url])
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
n=2,
max_completion_tokens=10,
extra_body=dict(use_beam_search=True),
)
assert len(chat_completion.choices) == 2
assert (
chat_completion.choices[0].message.content
!= chat_completion.choices[1].message.content
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
async def test_chat_streaming_video(
client: openai.AsyncOpenAI, model_name: str, video_url: str
):
messages = dummy_messages_from_video_url(video_url)
# test single completion
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
temperature=0.0,
)
output = chat_completion.choices[0].message.content
stop_reason = chat_completion.choices[0].finish_reason
# test streaming
stream = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
temperature=0.0,
stream=True,
)
chunks: list[str] = []
finish_reason_count = 0
async for chunk in stream:
delta = chunk.choices[0].delta
if delta.role:
assert delta.role == "assistant"
if delta.content:
chunks.append(delta.content)
if chunk.choices[0].finish_reason is not None:
finish_reason_count += 1
# finish reason should only return in last block
assert finish_reason_count == 1
assert chunk.choices[0].finish_reason == stop_reason
assert delta.content
assert "".join(chunks) == output
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize(
"video_urls", [TEST_VIDEO_URLS[:i] for i in range(2, len(TEST_VIDEO_URLS))]
)
@pytest.mark.flaky(
reruns=2,
reruns_delay=5,
condition=current_platform.is_rocm(),
)
async def test_multi_video_input(
client: openai.AsyncOpenAI, model_name: str, video_urls: list[str]
):
messages = dummy_messages_from_video_url(video_urls)
if len(video_urls) > MAXIMUM_VIDEOS:
with pytest.raises(openai.BadRequestError): # test multi-video input
await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
temperature=0.0,
)
# the server should still work afterwards
completion = await client.completions.create(
model=model_name,
prompt=[0, 0, 0, 0, 0],
max_tokens=5,
temperature=0.0,
)
completion = completion.choices[0].text
assert completion is not None and len(completion) >= 0
else:
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
temperature=0.0,
)
message = chat_completion.choices[0].message
assert message.content is not None and len(message.content) >= 0
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_omitted_video_url_requires_cached_uuid(
client: openai.AsyncOpenAI,
model_name: str,
):
video_uuid = "test-video-uuid"
video_url = TEST_VIDEO_URLS[0]
omitted_video_messages = describe_video_messages(
None,
extra_video_fields={"uuid": video_uuid},
)
with pytest.raises(
openai.BadRequestError,
match="Cache miss for video at index 0 but data is not provided",
):
await client.chat.completions.create(
model=model_name,
messages=omitted_video_messages,
)
await complete_and_check(
client,
model_name,
describe_video_messages(
video_url,
extra_video_fields={"uuid": video_uuid},
),
context="populate UUID cache",
)
await complete_and_check(
client,
model_name,
omitted_video_messages,
context="reuse cached video without URL",
)
@@ -1,676 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import openai
import pytest
import pytest_asyncio
from transformers import AutoProcessor
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
from vllm.multimodal.media import MediaWithBytes
from vllm.multimodal.utils import encode_image_url, fetch_image
from vllm.platforms import current_platform
MODEL_NAME = "microsoft/Phi-3.5-vision-instruct"
MAXIMUM_IMAGES = 2
# Required terms for beam search validation
# Each entry is a list of term groups - ALL groups must match
# Each group is a list of alternatives - at least ONE term in the group must appear
# This provides semantic validation while allowing wording variation
REQUIRED_BEAM_SEARCH_TERMS = [
# Boardwalk image: must have "boardwalk" AND ("wooden" or "wood")
[["boardwalk"], ["wooden", "wood"]],
# Parrots image: must have ("parrot" or "bird") AND "two"
[["parrot", "bird"], ["two"]],
# Venn diagram: must have "venn" AND "diagram"
[["venn"], ["diagram"]],
# Gradient image: must have "gradient" AND ("color" or "spectrum")
[["gradient"], ["color", "spectrum"]],
]
def check_output_matches_terms(content: str, term_groups: list[list[str]]) -> bool:
"""Check if content matches all required term groups.
Each term group requires at least one of its terms to be present.
All term groups must be satisfied.
"""
content_lower = content.lower()
return all(
any(term.lower() in content_lower for term in group) for group in term_groups
)
def assert_non_empty_content(chat_completion, *, context: str = "") -> str:
"""Assert the first choice has non-empty string content; return it.
Provides a detailed failure message including the full ChatCompletion
response so flaky / model-quality issues are easy to diagnose.
"""
prefix = f"[{context}] " if context else ""
choice = chat_completion.choices[0]
content = choice.message.content
assert content is not None, (
f"{prefix}Expected non-None content but got None. "
f"finish_reason={choice.finish_reason!r}, "
f"full message={choice.message!r}, "
f"usage={chat_completion.usage!r}"
)
assert isinstance(content, str), (
f"{prefix}Expected str content, got {type(content).__name__}: {content!r}"
)
assert len(content) > 0, (
f"{prefix}Expected non-empty content but got empty string. "
f"finish_reason={choice.finish_reason!r}, "
f"full message={choice.message!r}, "
f"usage={chat_completion.usage!r}"
)
return content
@pytest.fixture(scope="module")
def server():
args = [
"--runner",
"generate",
"--max-model-len",
"2048",
"--max-num-seqs",
"5",
"--enforce-eager",
"--trust-remote-code",
"--limit-mm-per-prompt",
json.dumps({"image": MAXIMUM_IMAGES}),
*ROCM_EXTRA_ARGS,
]
# ROCm: Allow more time to download videos from external sources.
env_overrides = {
**(
{
"VLLM_VIDEO_FETCH_TIMEOUT": "120",
}
if current_platform.is_rocm()
else {}
),
}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_overrides) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
@pytest.fixture(scope="session")
def url_encoded_image(local_asset_server) -> dict[str, str]:
return {
image_asset: encode_image_url(local_asset_server.get_image_asset(image_asset))
for image_asset in TEST_IMAGE_ASSETS
}
def dummy_messages_from_image_url(
image_urls: str | list[str],
content_text: str = "What's in this image?",
):
if isinstance(image_urls, str):
image_urls = [image_urls]
return [
{
"role": "user",
"content": [
*(
{"type": "image_url", "image_url": {"url": image_url}}
for image_url in image_urls
),
{"type": "text", "text": content_text},
],
}
]
def describe_image_messages(
image_url: str, *, extra_image_fields: dict | None = None
) -> list[dict]:
"""Build the system + user messages used by the completions-with-image
family of tests. *extra_image_fields* is merged into the top-level
image content block (for uuid / bad-key tests)."""
image_block: dict = {
"type": "image_url",
"image_url": {"url": image_url},
}
if extra_image_fields:
image_block.update(extra_image_fields)
return [
{"role": "system", "content": "You are a helpful assistant."},
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this image."},
image_block,
],
},
]
async def complete_and_check(
client: openai.AsyncOpenAI,
model_name: str,
messages: list[dict],
*,
context: str,
max_completion_tokens: int = 50,
temperature: float = 0.0,
) -> str:
"""Run a chat completion and assert the output is non-empty.
Returns the content string."""
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=max_completion_tokens,
temperature=temperature,
)
return assert_non_empty_content(chat_completion, context=context)
def get_hf_prompt_tokens(model_name, content, image_url):
processor = AutoProcessor.from_pretrained(
model_name, trust_remote_code=True, num_crops=4
)
placeholder = "<|image_1|>\n"
messages = [
{
"role": "user",
"content": f"{placeholder}{content}",
}
]
image = fetch_image(image_url)
# Unwrap MediaWithBytes if present
if isinstance(image, MediaWithBytes):
image = image.media
images = [image]
prompt = processor.tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
inputs = processor(prompt, images, return_tensors="pt")
return inputs.input_ids.shape[1]
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
async def test_single_chat_session_image(
client: openai.AsyncOpenAI, model_name: str, image_url: str
):
content_text = "What's in this image?"
messages = dummy_messages_from_image_url(image_url, content_text)
max_completion_tokens = 10
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=max_completion_tokens,
logprobs=True,
temperature=0.0,
top_logprobs=5,
)
assert len(chat_completion.choices) == 1, (
f"Expected 1 choice, got {len(chat_completion.choices)}"
)
choice = chat_completion.choices[0]
assert choice.finish_reason == "length", (
f"Expected finish_reason='length' (capped at {max_completion_tokens} "
f"tokens), got {choice.finish_reason!r}. "
f"content={choice.message.content!r}"
)
hf_prompt_tokens = get_hf_prompt_tokens(model_name, content_text, image_url)
expected_usage = openai.types.CompletionUsage(
completion_tokens=max_completion_tokens,
prompt_tokens=hf_prompt_tokens,
total_tokens=hf_prompt_tokens + max_completion_tokens,
)
assert chat_completion.usage == expected_usage, (
f"Usage mismatch: got {chat_completion.usage!r}, expected {expected_usage!r}"
)
message = choice.message
assert message.content is not None and len(message.content) >= 10, (
f"Expected content with >=10 chars, got {message.content!r}"
)
assert message.role == "assistant", (
f"Expected role='assistant', got {message.role!r}"
)
messages.append({"role": "assistant", "content": message.content})
# test multi-turn dialogue
messages.append({"role": "user", "content": "express your result in json"})
await complete_and_check(
client,
model_name,
messages,
context=f"multi-turn follow-up for {image_url}",
max_completion_tokens=10,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
async def test_error_on_invalid_image_url_type(
client: openai.AsyncOpenAI, model_name: str, image_url: str
):
content_text = "What's in this image?"
messages = [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": image_url},
{"type": "text", "text": content_text},
],
}
]
# image_url should be a dict {"url": "some url"}, not directly a string
with pytest.raises(openai.BadRequestError):
await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
temperature=0.0,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
async def test_single_chat_session_image_beamsearch(
client: openai.AsyncOpenAI, model_name: str, image_url: str
):
content_text = "What's in this image?"
messages = dummy_messages_from_image_url(image_url, content_text)
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
n=2,
max_completion_tokens=10,
logprobs=True,
top_logprobs=5,
extra_body=dict(use_beam_search=True),
)
assert len(chat_completion.choices) == 2, (
f"Expected 2 beam search choices, got {len(chat_completion.choices)}"
)
content_0 = chat_completion.choices[0].message.content
content_1 = chat_completion.choices[1].message.content
assert content_0 != content_1, (
f"Beam search should produce different outputs for {image_url}, "
f"but both returned: {content_0!r}"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("raw_image_url", TEST_IMAGE_ASSETS)
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
async def test_single_chat_session_image_base64encoded(
client: openai.AsyncOpenAI,
model_name: str,
raw_image_url: str,
image_url: str,
url_encoded_image: dict[str, str],
):
content_text = "What's in this image?"
messages = dummy_messages_from_image_url(
url_encoded_image[raw_image_url],
content_text,
)
max_completion_tokens = 10
# test single completion
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=max_completion_tokens,
logprobs=True,
temperature=0.0,
top_logprobs=5,
)
assert len(chat_completion.choices) == 1, (
f"Expected 1 choice, got {len(chat_completion.choices)}"
)
choice = chat_completion.choices[0]
assert choice.finish_reason == "length", (
f"Expected finish_reason='length', got {choice.finish_reason!r}. "
f"content={choice.message.content!r}"
)
hf_prompt_tokens = get_hf_prompt_tokens(model_name, content_text, image_url)
expected_usage = openai.types.CompletionUsage(
completion_tokens=max_completion_tokens,
prompt_tokens=hf_prompt_tokens,
total_tokens=hf_prompt_tokens + max_completion_tokens,
)
assert chat_completion.usage == expected_usage, (
f"Usage mismatch: got {chat_completion.usage!r}, expected {expected_usage!r}"
)
message = choice.message
assert message.content is not None and len(message.content) >= 10, (
f"Expected content with >=10 chars, got {message.content!r}"
)
assert message.role == "assistant", (
f"Expected role='assistant', got {message.role!r}"
)
messages.append({"role": "assistant", "content": message.content})
# test multi-turn dialogue
messages.append({"role": "user", "content": "express your result in json"})
await complete_and_check(
client,
model_name,
messages,
context=f"multi-turn base64 follow-up for {raw_image_url}",
max_completion_tokens=10,
temperature=0.0,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("image_idx", list(range(len(TEST_IMAGE_ASSETS))))
async def test_single_chat_session_image_base64encoded_beamsearch(
client: openai.AsyncOpenAI,
model_name: str,
image_idx: int,
url_encoded_image: dict[str, str],
):
# NOTE: This test validates that we pass MM data through beam search
raw_image_url = TEST_IMAGE_ASSETS[image_idx]
required_terms = REQUIRED_BEAM_SEARCH_TERMS[image_idx]
messages = dummy_messages_from_image_url(url_encoded_image[raw_image_url])
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
n=2,
max_completion_tokens=10,
temperature=0.0,
extra_body=dict(use_beam_search=True),
)
assert len(chat_completion.choices) == 2, (
f"Expected 2 beam search choices for image {image_idx} "
f"({raw_image_url}), got {len(chat_completion.choices)}"
)
# Verify beam search produces two different non-empty outputs
content_0 = chat_completion.choices[0].message.content
content_1 = chat_completion.choices[1].message.content
# Emit beam search outputs for debugging
print(
f"Beam search outputs for image {image_idx} ({raw_image_url}): "
f"Output 0: {content_0!r}, Output 1: {content_1!r}"
)
assert content_0, (
f"First beam output is empty for image {image_idx} ({raw_image_url}). "
f"finish_reason={chat_completion.choices[0].finish_reason!r}"
)
assert content_1, (
f"Second beam output is empty for image {image_idx} "
f"({raw_image_url}). "
f"finish_reason={chat_completion.choices[1].finish_reason!r}"
)
assert content_0 != content_1, (
f"Beam search produced identical outputs for image {image_idx} "
f"({raw_image_url}): {content_0!r}"
)
# Verify each output contains the required terms for this image
for i, content in enumerate([content_0, content_1]):
assert check_output_matches_terms(content, required_terms), (
f"Beam output {i} for image {image_idx} ({raw_image_url}) "
f"doesn't match required terms.\n"
f" content: {content!r}\n"
f" required (all groups, >=1 per group): {required_terms}"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
async def test_chat_streaming_image(
client: openai.AsyncOpenAI, model_name: str, image_url: str
):
messages = dummy_messages_from_image_url(image_url)
# test single completion
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
temperature=0.0,
)
output = chat_completion.choices[0].message.content
stop_reason = chat_completion.choices[0].finish_reason
# test streaming
stream = await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
temperature=0.0,
stream=True,
)
chunks: list[str] = []
finish_reason_count = 0
async for chunk in stream:
delta = chunk.choices[0].delta
if delta.role:
assert delta.role == "assistant", (
f"Expected role='assistant' in stream delta, got {delta.role!r}"
)
if delta.content:
chunks.append(delta.content)
if chunk.choices[0].finish_reason is not None:
finish_reason_count += 1
# finish reason should only return in last block
assert finish_reason_count == 1, (
f"Expected exactly 1 finish_reason across stream chunks, "
f"got {finish_reason_count}"
)
assert chunk.choices[0].finish_reason == stop_reason, (
f"Stream finish_reason={chunk.choices[0].finish_reason!r} "
f"doesn't match non-stream finish_reason={stop_reason!r}"
)
streamed_text = "".join(chunks)
assert streamed_text == output, (
f"Streamed output doesn't match non-streamed for {image_url}.\n"
f" streamed: {streamed_text!r}\n"
f" non-streamed: {output!r}"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize(
"image_urls",
[TEST_IMAGE_ASSETS[:i] for i in range(2, len(TEST_IMAGE_ASSETS))],
indirect=True,
)
async def test_multi_image_input(
client: openai.AsyncOpenAI, model_name: str, image_urls: list[str]
):
messages = dummy_messages_from_image_url(image_urls)
if len(image_urls) > MAXIMUM_IMAGES:
with pytest.raises(openai.BadRequestError): # test multi-image input
await client.chat.completions.create(
model=model_name,
messages=messages,
max_completion_tokens=10,
temperature=0.0,
)
# the server should still work afterwards
completion = await client.completions.create(
model=model_name,
prompt=[0, 0, 0, 0, 0],
max_tokens=5,
temperature=0.0,
)
assert completion.choices[0].text is not None, (
"Server failed to produce output after rejecting over-limit "
"multi-image request"
)
else:
await complete_and_check(
client,
model_name,
messages,
context=f"multi-image input ({len(image_urls)} images)",
max_completion_tokens=10,
temperature=0.0,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize(
"image_urls",
[TEST_IMAGE_ASSETS[:i] for i in range(2, len(TEST_IMAGE_ASSETS))],
indirect=True,
)
async def test_completions_with_image(
client: openai.AsyncOpenAI,
model_name: str,
image_urls: list[str],
):
for image_url in image_urls:
messages = describe_image_messages(image_url)
await complete_and_check(
client,
model_name,
messages,
context=f"completions_with_image url={image_url}",
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize(
"image_urls",
[TEST_IMAGE_ASSETS[:i] for i in range(2, len(TEST_IMAGE_ASSETS))],
indirect=True,
)
async def test_completions_with_image_with_uuid(
client: openai.AsyncOpenAI,
model_name: str,
image_urls: list[str],
):
for image_url in image_urls:
messages = describe_image_messages(
image_url,
extra_image_fields={"uuid": image_url},
)
await complete_and_check(
client,
model_name,
messages,
context=f"uuid first request url={image_url}",
)
cached_messages: list[dict] = [
{"role": "system", "content": "You are a helpful assistant."},
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this image."},
{"type": "image_url", "image_url": {}, "uuid": image_url},
],
},
]
await complete_and_check(
client,
model_name,
cached_messages,
context=f"uuid cached (empty image) uuid={image_url}",
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_completions_with_empty_image_with_uuid_without_cache_hit(
client: openai.AsyncOpenAI,
model_name: str,
):
with pytest.raises(openai.BadRequestError):
await client.chat.completions.create(
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this image."},
{
"type": "image_url",
"image_url": {},
"uuid": "uuid_not_previously_seen",
},
],
},
],
model=model_name,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize(
"image_urls",
[TEST_IMAGE_ASSETS[:i] for i in range(2, len(TEST_IMAGE_ASSETS))],
indirect=True,
)
async def test_completions_with_image_with_incorrect_uuid_format(
client: openai.AsyncOpenAI,
model_name: str,
image_urls: list[str],
):
for image_url in image_urls:
messages = describe_image_messages(
image_url,
extra_image_fields={
"also_incorrect_uuid_key": image_url,
},
)
# Inject the bad key inside image_url dict too
messages[1]["content"][1]["image_url"]["incorrect_uuid_key"] = image_url
await complete_and_check(
client,
model_name,
messages,
context=f"incorrect uuid format url={image_url}",
)