[Test] Split the chat_utils tests. (#61765)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-10-15 08:26:22 +00:00
committed by GitHub
parent b965ee564d
commit 0f0bc4dbb1
15 changed files with 3328 additions and 3242 deletions
+27
View File
@@ -0,0 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.assets.audio import AudioAsset
from vllm.assets.image import ImageAsset
from vllm.assets.video import VideoAsset
from vllm.multimodal.utils import encode_audio_url, encode_image_url, encode_video_url
@pytest.fixture(scope="module")
def image_url():
image = ImageAsset("cherry_blossom")
return encode_image_url(image.pil_image)
@pytest.fixture(scope="module")
def video_url():
video = VideoAsset("baby_reading", 1)
return encode_video_url(video.np_ndarrays)
@pytest.fixture(scope="module")
def audio_url():
audio = AudioAsset("mary_had_lamb")
return encode_audio_url(*audio.audio_and_sample_rate)
@@ -0,0 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
from vllm.renderers.chat_utils import ConversationMessage, parse_chat_message_content
def _parse_one(message) -> ConversationMessage:
# Plain-text content only reads these two tracker attributes.
tracker = MagicMock()
parser = tracker.create_parser.return_value
parser.model_config.enable_prompt_embeds = False
parser.mm_placeholder_storage.return_value = {}
result = parse_chat_message_content(
message, tracker, content_format="string", interleave_strings=False
)
assert len(result) == 1
return result[0]
def test_assistant_reasoning_is_mirrored_to_reasoning_content():
"""``reasoning`` is mirrored to ``reasoning_content`` for legacy templates."""
conv_msg = _parse_one(
{
"role": "assistant",
"content": "4",
"reasoning": "2+2 equals 4",
}
)
assert conv_msg.get("reasoning") == "2+2 equals 4"
assert conv_msg.get("reasoning_content") == "2+2 equals 4"
def test_assistant_without_reasoning_sets_neither_field():
"""No reasoning in, no reasoning keys out: no empty thinking block in templates."""
conv_msg = _parse_one({"role": "assistant", "content": "Hi"})
assert "reasoning" not in conv_msg
assert "reasoning_content" not in conv_msg
def test_non_assistant_reasoning_is_not_mirrored():
"""Only assistant turns carry reasoning; a user turn must not."""
conv_msg = _parse_one(
{"role": "user", "content": "Hi", "reasoning": "should be ignored"}
)
assert "reasoning" not in conv_msg
assert "reasoning_content" not in conv_msg
+224
View File
@@ -0,0 +1,224 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.config import ModelConfig
from vllm.renderers.chat_utils import parse_chat_messages, parse_chat_messages_async
from vllm.utils.serial_utils import tensor2base64
from .utils import (
assert_mm_data_inputs,
assert_mm_uuids,
)
QWEN2AUDIO_MODEL_ID = "Qwen/Qwen2-Audio-7B-Instruct"
@pytest.fixture(scope="function")
def qwen2_audio_model_config():
return ModelConfig(
QWEN2AUDIO_MODEL_ID,
runner="generate",
trust_remote_code=True,
limit_mm_per_prompt={
"audio": 1,
},
)
@pytest.fixture(scope="function")
def audio_embeds_model_config():
return ModelConfig(
QWEN2AUDIO_MODEL_ID,
runner="generate",
trust_remote_code=True,
limit_mm_per_prompt={
"audio": 2,
},
enable_mm_embeds=True,
)
@pytest.mark.parametrize("input_audio", [None, {}, {"data": "", "format": "wav"}])
def test_parse_chat_messages_single_empty_audio_with_uuid(
qwen2_audio_model_config,
input_audio,
):
audio_uuid = "abcd"
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{
"type": "input_audio",
"input_audio": input_audio,
"uuid": audio_uuid,
},
{"type": "text", "text": "What does the audio say?"},
],
}
],
qwen2_audio_model_config,
content_format="string",
)
assert conversation == [
{
"role": "user",
"content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the "
"audio say?",
}
]
assert_mm_data_inputs(mm_data, {"audio": 1})
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
@pytest.mark.asyncio
@pytest.mark.parametrize("input_audio", [None, {}, {"data": "", "format": "wav"}])
async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
qwen2_audio_model_config,
input_audio,
):
audio_uuid = "abcd"
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
[
{
"role": "user",
"content": [
{
"type": "input_audio",
"input_audio": input_audio,
"uuid": audio_uuid,
},
{"type": "text", "text": "What does the audio say?"},
],
}
],
qwen2_audio_model_config,
content_format="string",
)
assert conversation == [
{
"role": "user",
"content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the "
"audio say?",
}
]
assert_mm_data_inputs(mm_data, {"audio": 1})
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
def test_parse_chat_messages_empty_audio_embeds_with_uuid(
audio_embeds_model_config,
):
"""Test audio_embeds with UUID (no actual embeds data)."""
uuid = "test-audio-uuid-123"
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this audio"},
{"type": "audio_embeds", "audio_embeds": None, "uuid": uuid},
],
}
],
audio_embeds_model_config,
content_format="string",
)
# Should have audio in mm_data as None (UUID provided)
assert mm_data is not None
assert "audio" in mm_data
assert isinstance(mm_data["audio"], list)
assert len(mm_data["audio"]) == 1
assert mm_data["audio"][0] is None
# UUID should be recorded
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[uuid])
def test_parse_chat_messages_audio_embeds_with_string(
audio_embeds_model_config,
):
"""Test audio_embeds with base64 string embedding data."""
import torch
# Create a sample audio embedding tensor
hidden_size = audio_embeds_model_config.get_inputs_embeds_size()
audio_embedding = torch.randn(1, 128, hidden_size)
# Encode it as base64
base64_audio_embedding = tensor2base64(audio_embedding)
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this audio"},
{
"type": "audio_embeds",
"audio_embeds": base64_audio_embedding,
},
],
}
],
audio_embeds_model_config,
content_format="string",
)
# Should have audio embedding in mm_data (single tensor, not a list)
assert mm_data is not None
assert "audio" in mm_data
assert isinstance(mm_data["audio"], torch.Tensor)
assert mm_data["audio"].shape == audio_embedding.shape
# No UUID provided
assert mm_uuids is not None
assert "audio" in mm_uuids
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
@pytest.mark.asyncio
async def test_parse_chat_messages_audio_embeds_async(
audio_embeds_model_config,
):
"""Test audio_embeds with async futures."""
import torch
# Create a sample audio embedding tensor
hidden_size = audio_embeds_model_config.get_inputs_embeds_size()
audio_embedding = torch.randn(1, 128, hidden_size)
# Encode it as base64
base64_audio_embedding = tensor2base64(audio_embedding)
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
[
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this audio"},
{
"type": "audio_embeds",
"audio_embeds": base64_audio_embedding,
},
],
}
],
audio_embeds_model_config,
content_format="string",
)
# Should have audio embedding in mm_data (single tensor, not a list)
assert mm_data is not None
assert "audio" in mm_data
assert isinstance(mm_data["audio"], torch.Tensor)
assert mm_data["audio"].shape == audio_embedding.shape
# No UUID provided
assert mm_uuids is not None
assert "audio" in mm_uuids
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
@@ -0,0 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.exceptions import VLLMValidationError
from vllm.renderers.chat_utils import validate_chat_template
def test_validate_chat_template_rejects_invalid_type():
"""A non-str/Path chat_template is invalid user input."""
with pytest.raises(
VLLMValidationError, match="not a valid chat template type"
) as exc_info:
validate_chat_template(123)
assert exc_info.value.parameter == "chat_template"
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from unittest.mock import MagicMock
import pytest
from vllm.renderers.chat_utils import AsyncMultiModalItemTracker
@pytest.mark.asyncio
async def test_resolve_items_runs_modalities_concurrently_and_preserves_order():
"""Media fetches overlap while modality and item order are preserved."""
active_fetches = 0
max_active_fetches = 0
async def _fetch(name: str, delay: float):
nonlocal active_fetches, max_active_fetches
active_fetches += 1
max_active_fetches = max(max_active_fetches, active_fetches)
try:
await asyncio.sleep(delay)
return name, f"{name}-uuid"
finally:
active_fetches -= 1
tracker = AsyncMultiModalItemTracker(MagicMock(is_multimodal_model=True))
tracker.__dict__["mm_processor"] = MagicMock()
tracker._items_by_modality["video"] = [
lambda: _fetch("video-0", 0.02),
lambda: _fetch("video-1", 0.01),
]
tracker._items_by_modality["image"] = [
lambda: _fetch("image-0", 0.02),
lambda: _fetch("image-1", 0.01),
]
tracker._items_by_modality["audio"] = [
lambda: _fetch("audio-0", 0.02),
lambda: _fetch("audio-1", 0.01),
]
mm_data, mm_uuids = await tracker.resolve_items()
assert mm_data is not None
assert mm_uuids is not None
assert max_active_fetches == 6
assert list(mm_data) == ["image", "audio", "video"]
assert list(mm_uuids) == ["image", "audio", "video"]
assert mm_data == {
"image": ["image-0", "image-1"],
"audio": ["audio-0", "audio-1"],
"video": ["video-0", "video-1"],
}
assert mm_uuids == {
"image": ["image-0-uuid", "image-1-uuid"],
"audio": ["audio-0-uuid", "audio-1-uuid"],
"video": ["video-0-uuid", "video-1-uuid"],
}
@pytest.mark.asyncio
async def test_resolve_items_does_not_leak_tasks_on_partial_failure():
"""Regression test: one failing media fetch must not abandon the other
still-in-flight fetches across modality batches.
Before the fix, `resolve_items` gathered per-modality fetches with plain
`asyncio.gather`, so the first exception propagated immediately while
sibling fetches (real network/thread-pool work in production) kept
running detached, with nothing left to await or cancel them.
"""
async def _fetch(should_fail: bool, delay: float):
if should_fail:
await asyncio.sleep(0.01)
raise ValueError("simulated fetch failure")
await asyncio.sleep(delay)
return ("decoded", None)
tracker = AsyncMultiModalItemTracker(MagicMock())
tracker._items_by_modality["image"] = [
lambda: _fetch(True, 0),
lambda: _fetch(False, 0.2),
lambda: _fetch(False, 0.2),
]
tracker._items_by_modality["audio"] = [lambda: _fetch(False, 0.2)]
tracker._items_by_modality["video"] = [lambda: _fetch(False, 0.2)]
tasks_before = asyncio.all_tasks()
with pytest.raises(ValueError, match="simulated fetch failure"):
await tracker.resolve_items()
leaked_tasks = asyncio.all_tasks() - tasks_before
assert not leaked_tasks, (
f"resolve_items left {len(leaked_tasks)} task(s) running after "
f"raising: {leaked_tasks}"
)
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.exceptions import VLLMValidationError
from vllm.renderers.chat_utils.content_parser import _load_embeds_dict
from vllm.renderers.chat_utils.item_tracker import _parse_metadata_array
@pytest.mark.parametrize("values", [[1, 32, 48], [0.0, 0.12345678912345678]])
def test_json_metadata_preserves_numeric_precision(values):
tensor = _parse_metadata_array("grid", values, {"grid"})
assert tensor.tolist() == values
assert tensor.dtype == (
torch.float64 if isinstance(values[0], float) else torch.int64
)
@pytest.mark.parametrize(
"key,values",
[
("image_embeds", [1, 2]),
("grid", [True]),
("grid", [float("nan")]),
("grid", [float("inf")]),
("grid", [[1, 2]]),
("grid", [2**64]),
],
)
def test_json_arrays_are_only_valid_numeric_metadata(key, values):
with pytest.raises(VLLMValidationError):
_parse_metadata_array(key, values, {"grid"})
@pytest.mark.asyncio
async def test_json_metadata_bypasses_tensor_deserialization():
from unittest.mock import AsyncMock
tensor = torch.ones(2, 3)
fetch = AsyncMock(return_value=tensor)
result = await _load_embeds_dict(
{"image_embeds": "legacy-base64", "image_grid_thw": [1, 2, 2]}, fetch
)
fetch.assert_awaited_once_with("legacy-base64")
assert result["image_embeds"] is tensor
assert result["image_grid_thw"] == [1, 2, 2]
@@ -0,0 +1,625 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Offline unit tests for `prompt_embeds` chat-completion content parts."""
from __future__ import annotations
import inspect
import io
from typing import Final
from unittest import mock
import pybase64 as base64
import pytest
import regex as re
import torch
from transformers import AutoTokenizer
from vllm.exceptions import VLLMValidationError
from vllm.renderers import TokenizeParams
from vllm.renderers.chat_utils import parse_chat_messages, parse_chat_messages_async
from vllm.renderers.chat_utils.chat_message import (
_PROMPT_EMBEDS_MISSING_DATA_ERROR,
_RESERVED_PLACEHOLDER_IN_TEXT_ERROR,
MM_PARSER_MAP,
)
from vllm.renderers.chat_utils.content_parser import (
_ENABLE_PROMPT_EMBEDS_ERROR,
MODALITY_PLACEHOLDERS_MAP,
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
)
from vllm.renderers.hf import (
_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR,
_build_mixed_prompt_embeds,
_build_prompt_embeds_positions,
_build_prompt_embeds_updates,
_ensure_prompt_embeds_placeholder_token,
_expand_prompt_embeds_placeholders,
)
# Cover distinct tokenizer families:
# GPT2TokenizerFast (BPE, OpenAI-style)
# Qwen2TokenizerFast (SentencePiece BPE variant)
# BertTokenizerFast (WordPiece)
TOKENIZER_IDS: Final[list[str]] = [
"openai-community/gpt2",
"Qwen/Qwen2.5-1.5B-Instruct",
"bert-base-uncased",
]
@pytest.fixture(params=TOKENIZER_IDS, ids=TOKENIZER_IDS)
def tokenizer(request):
"""A fresh tokenizer instance per tokenizer family."""
return AutoTokenizer.from_pretrained(request.param)
# Minimal chat template that works with any tokenizer. Iterates
# `message.content` as either a string or a list of dicts (openai format).
_SIMPLE_CHAT_TEMPLATE: Final[str] = (
"{% for m in messages %}"
"{% if m['content'] is string %}{{m['content']}}"
"{% else %}{% for p in m['content'] %}{{p['text']}}{% endfor %}"
"{% endif %}\n{% endfor %}"
)
async def _maybe_await(fn, *args, **kwargs):
"""Call *fn* and `await` the result if it's a coroutine."""
result = fn(*args, **kwargs)
if inspect.iscoroutine(result):
result = await result
return result
# Parametrize over sync / async parse paths so every end-to-end test
# exercises both.
_PARSE_FUNCTIONS = [parse_chat_messages, parse_chat_messages_async]
@pytest.fixture(params=_PARSE_FUNCTIONS, ids=["sync", "async"])
def parse_fn(request):
"""Either the sync or async `parse_chat_messages` callable."""
return request.param
def _encode_tensor(t: torch.Tensor) -> str:
buf = io.BytesIO()
torch.save(t, buf)
return base64.b64encode(buf.getvalue()).decode("utf-8")
_MOCK_HIDDEN_SIZE: Final[int] = 8
_MOCK_DTYPE: Final[torch.dtype] = torch.float32
def _make_mock_model_config(*, enable_prompt_embeds: bool = True) -> mock.MagicMock:
mc = mock.MagicMock()
mc.enable_prompt_embeds = enable_prompt_embeds
mc.multimodal_config = None
mc.allowed_local_media_path = None
mc.allowed_media_domains = None
# Test text-only code path in `MultiModalItemTracker.resolve_items`.
mc.is_multimodal_model = False
# `safe_load_prompt_embeds` pins each tensor to the model's hidden_size
# and dtype, so the mock must return concrete values.
mc.get_hidden_size.return_value = _MOCK_HIDDEN_SIZE
mc.dtype = _MOCK_DTYPE
return mc
def test_prompt_embeds_keys_registered():
assert "prompt_embeds" in MODALITY_PLACEHOLDERS_MAP
assert MODALITY_PLACEHOLDERS_MAP["prompt_embeds"] == "<##PROMPT_EMBEDS##>"
assert "prompt_embeds" in MM_PARSER_MAP
def test_ensure_placeholder_token_is_single_token_and_idempotent(tokenizer):
"""Ensure the placeholder token is a single token and that multiple calls to
"ensure" are idempotent, across all tokenizer families."""
tid1 = _ensure_prompt_embeds_placeholder_token(tokenizer)
tid2 = _ensure_prompt_embeds_placeholder_token(tokenizer)
assert tid1 == tid2
ids = tokenizer.encode(PROMPT_EMBEDS_PLACEHOLDER_TOKEN, add_special_tokens=False)
assert ids == [tid1]
# Repeating it in a string N times must produce exactly that many tokens.
N = 5
ids_rep = tokenizer.encode(
PROMPT_EMBEDS_PLACEHOLDER_TOKEN * N, add_special_tokens=False
)
assert ids_rep == [tid1] * N
def test_parse_chat_messages_openai_format():
NUM_TOKENS = 3
t = torch.randn(NUM_TOKENS, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
b64 = _encode_tensor(t)
mc = _make_mock_model_config()
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Hello "},
{"type": "prompt_embeds", "data": b64},
{"type": "text", "text": " world"},
],
}
]
conv, mm_data, _ = parse_chat_messages(
messages,
mc,
content_format="openai",
)
# The middle content part is rewritten to a single placeholder-token
# sentinel.
texts = [p["text"] for p in conv[0]["content"]]
assert texts == [
"Hello ",
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
" world",
]
assert mm_data is not None and "prompt_embeds" in mm_data
assert torch.equal(mm_data["prompt_embeds"][0], t)
# Each layout entry is one content part:
# ("text", "A") -> {"type": "text", "text": "A"}
# ("embed", N) -> {"type": "prompt_embeds", "data": <base64 of (N, H) tensor>}
@pytest.mark.parametrize(
"layout",
[
# Case: Single embed only.
[("embed", 2)],
# Case: Embed at the start of the message.
[("embed", 3), ("text", "B")],
# Case: Embed at the end of the message.
[("text", "A"), ("embed", 1)],
# Case: Embed sandwiched between text spans.
[("text", "A"), ("embed", 2), ("text", "B")],
# Case: Multiple embeds with text in between.
[("text", "A"), ("embed", 2), ("text", "B"), ("embed", 3)],
# Case: Adjacent embeds with no separating text.
[("embed", 1), ("embed", 2)],
# Case: Multiple text spans before a trailing embed.
[("text", "A"), ("text", "B"), ("embed", 1)],
# Case: Long-ish run mixing both kinds.
[
("text", "head"),
("embed", 4),
("text", "mid"),
("embed", 1),
("embed", 2),
("text", "tail"),
],
],
ids=[
"single-embed",
"embed-then-text",
"text-then-embed",
"text-embed-text",
"text-embed-text-embed",
"adjacent-embeds",
"text-text-embed",
"long-mixed-run",
],
)
@pytest.mark.parametrize(
"interleave_mm_strings",
# `None`: text-only path where `multimodal_config` is absent.
# `False`: non-interleave multimodal path (the common default).
# `True`: sentinel-substitution interleave path.
# All three must preserve the request ordering of prompt_embeds
# relative to surrounding text because prompt_embeds are spliced at the
# token offset during rendering.
[None, False, True],
ids=["text-only", "interleave-off", "interleave-on"],
)
def test_parse_chat_messages_string_format_preserves_position(
layout, interleave_mm_strings
):
mc = _make_mock_model_config()
if interleave_mm_strings is not None:
mm_cfg = mock.MagicMock()
mm_cfg.interleave_mm_strings = interleave_mm_strings
mc.multimodal_config = mm_cfg
content: list[dict] = []
expected_parts: list[str] = []
expected_embeds: list[torch.Tensor] = []
for kind, value in layout:
if kind == "text":
content.append({"type": "text", "text": value})
expected_parts.append(value)
else: # prompt embeds
num_tokens = value
t = torch.randn(num_tokens, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
expected_embeds.append(t)
content.append({"type": "prompt_embeds", "data": _encode_tensor(t)})
# Parser emits ONE sentinel per part.
expected_parts.append(PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
messages = [{"role": "user", "content": content}]
conv, mm_data, _ = parse_chat_messages(
messages,
mc,
content_format="string",
)
assert conv[0]["content"] == "\n".join(expected_parts)
assert mm_data is not None and "prompt_embeds" in mm_data
assert len(mm_data["prompt_embeds"]) == len(expected_embeds)
for got, want in zip(mm_data["prompt_embeds"], expected_embeds, strict=True):
assert torch.equal(got, want)
def test_parse_chat_messages_requires_flag():
t = torch.randn(2, 4)
b64 = _encode_tensor(t)
mc = _make_mock_model_config(enable_prompt_embeds=False)
messages = [
{
"role": "user",
"content": [{"type": "prompt_embeds", "data": b64}],
}
]
with pytest.raises(VLLMValidationError, match=_ENABLE_PROMPT_EMBEDS_ERROR):
parse_chat_messages(
messages,
mc,
content_format="openai",
)
def test_parse_chat_messages_rejects_missing_data():
# `data` is marked `Required` on `ChatCompletionContentPartPromptEmbedsParam`;
# malformed requests without `data` must surface a clear validation error
# rather than being silently dropped.
mc = _make_mock_model_config()
messages = [
{
"role": "user",
"content": [{"type": "prompt_embeds"}], # no `data`
}
]
with pytest.raises(VLLMValidationError, match=_PROMPT_EMBEDS_MISSING_DATA_ERROR):
parse_chat_messages(
messages,
mc,
content_format="openai",
)
# Reserved placeholder guard: when `enable_prompt_embeds=True` the tokenizer is
# mutated to make `<prompt_embeds>` a single unsplittable token. Any user text
# containing that literal sequence would tokenize to the same sentinel ID and
# be mistaken for a splice point, so we reject it at parse time.
_PLACEHOLDER_ERROR_PATTERN: Final[str] = re.sub(
r"\\{[^}]*\\}", ".*", re.escape(_RESERVED_PLACEHOLDER_IN_TEXT_ERROR)
)
@pytest.mark.parametrize(
"content",
[
# Case: Top-level string content (wrapped as a single text part).
f"hello {PROMPT_EMBEDS_PLACEHOLDER_TOKEN} world",
# Case: List with a typed text part containing the placeholder.
[{"type": "text", "text": f"leading {PROMPT_EMBEDS_PLACEHOLDER_TOKEN}"}],
# Case: List with a plain-string part (no wrapping dict).
[f"raw string {PROMPT_EMBEDS_PLACEHOLDER_TOKEN}"],
],
ids=["top-level-string", "typed-text-part", "plain-string-part"],
)
def test_parse_chat_messages_rejects_placeholder_in_user_text(content):
mc = _make_mock_model_config() # enable_prompt_embeds=True by default
messages = [{"role": "user", "content": content}]
with pytest.raises(VLLMValidationError, match=_PLACEHOLDER_ERROR_PATTERN):
parse_chat_messages(messages, mc, content_format="openai")
def test_parse_chat_messages_allows_placeholder_in_text_when_feature_disabled():
# When `enable_prompt_embeds=False` the tokenizer is never mutated, so the
# literal `<prompt_embeds>` is just ordinary text and must pass through.
mc = _make_mock_model_config(enable_prompt_embeds=False)
messages = [
{
"role": "user",
"content": f"benign mention of {PROMPT_EMBEDS_PLACEHOLDER_TOKEN} here",
}
]
conv, mm_data, _ = parse_chat_messages(messages, mc, content_format="openai")
assert mm_data is None or "prompt_embeds" not in mm_data
# Text reaches the rendered conversation unchanged.
texts = [p["text"] for p in conv[0]["content"]]
assert PROMPT_EMBEDS_PLACEHOLDER_TOKEN in "".join(texts)
# Token-stream spec: ints are regular token IDs, tuples `(N,)` expand to
# a placeholder span of length N (creates corresponding `(N, H)` tensor).
# `expected` lists the `(start_idx, length)` pairs that
# `_build_prompt_embeds_positions` should return.
@pytest.mark.parametrize(
"stream, expected",
[
# Case: Single run in the middle.
([10, 20, (3,), 30], [(2, 3)]),
# Case: Single run at the start.
([(2,), 10, 20], [(0, 2)]),
# Case: Single run at the end.
([10, 20, (4,)], [(2, 4)]),
# Case: Two runs with tokens between.
([1, (2,), 2, 3, (3,), 4], [(1, 2), (5, 3)]),
# Case: Adjacent runs (no separating tokens).
([(1,), (2,)], [(0, 1), (1, 2)]),
# Case: Three runs.
([5, (2,), 6, (1,), 7, (3,), 8], [(1, 2), (4, 1), (6, 3)]),
],
ids=[
"single-middle",
"single-start",
"single-end",
"two-runs-separated",
"two-runs-adjacent",
"three-runs",
],
)
def test_build_positions(tokenizer, stream, expected):
H = 4
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
tensors: list[torch.Tensor] = []
token_ids: list[int] = []
for item in stream:
if isinstance(item, tuple):
length = item[0]
tensors.append(torch.randn(length, H))
token_ids.extend([tid] * length)
else:
token_ids.append(item)
mm_updates = _build_prompt_embeds_updates(tensors, tid)
positions = _build_prompt_embeds_positions(token_ids, len(tensors), mm_updates)
assert positions == expected
def test_build_positions_length_mismatch(tokenizer):
N1, H1 = 2, 4
N2, H2 = 3, 4
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
# 2 tensors expected but only a single placeholder run in the token
# stream (simulating dropping the second one).
tensors = [torch.randn(N1, H1), torch.randn(N2, H2)]
token_ids = [1, tid, tid, 2, 3]
mm_updates = _build_prompt_embeds_updates(tensors, tid)
# The error constant is a `str.format` template, escape it and turn
# the `{field}` placeholders into `.*` so it matches any substitution.
pattern = re.sub(
r"\\{[^}]*\\}", ".*", re.escape(_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR)
)
with pytest.raises(ValueError, match=pattern):
_build_prompt_embeds_positions(token_ids, len(tensors), mm_updates)
# ints = regular token IDs (any value)
# (N,) = embed span of length N
@pytest.mark.parametrize(
"stream",
[
[10, 20, (3,), 30],
[(2,), 10, 20],
[10, 20, (4,)],
[1, (2,), 2, 3, (3,), 4],
[(1,), (2,)],
[5, (2,), 6, (1,), 7, (3,), 8],
],
ids=[
"single-middle",
"single-start",
"single-end",
"two-spans-separated",
"two-spans-adjacent",
"three-spans",
],
)
def test_build_mixed_prompt_embeds(stream):
H = 8
_PLACEHOLDER = 0 # sentinel for embed positions in token_ids
tensors: list[torch.Tensor] = []
token_ids: list[int] = []
positions: list[tuple[int, int]] = []
cursor = 0
for item in stream:
if isinstance(item, tuple):
length = item[0]
tensors.append(torch.randn(length, H))
positions.append((cursor, length))
token_ids.extend([_PLACEHOLDER] * length)
cursor += length
else:
token_ids.append(item)
cursor += 1
embeds, mask = _build_mixed_prompt_embeds(token_ids, tensors, positions)
assert embeds.shape == (len(token_ids), H)
assert len(mask) == len(token_ids)
# Mask: False exactly at embed positions, True everywhere else.
expected_mask = torch.ones(len(token_ids), dtype=torch.bool)
for start, length in positions:
expected_mask[start : start + length] = False
assert mask == expected_mask.tolist()
# Embed rows match input tensors at the right positions.
for tensor, (start, length) in zip(tensors, positions):
assert torch.equal(embeds[start : start + length], tensor)
# Non-embed positions remain zero-filled.
assert torch.all(embeds[expected_mask] == 0)
# End-to-end tests: each runs both sync and async parse paths via the
# `parse_fn` fixture.
@pytest.mark.asyncio
@pytest.mark.parametrize("role", ["user", "system"])
async def test_end_to_end_expand_and_build(tokenizer, parse_fn, role):
"""Full renderer pipeline: parse -> chat template -> expand -> locate
-> build mixed prompt, across tokenizers, roles, and sync/async."""
tokenizer.chat_template = _SIMPLE_CHAT_TEMPLATE
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
LEN_A, LEN_B = 3, 2
t_a = torch.randn(LEN_A, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
t_b = torch.randn(LEN_B, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
NUM_TENSORS = 2
mc = _make_mock_model_config()
messages = [
{
"role": role,
"content": [
{"type": "text", "text": "Hello "},
{"type": "prompt_embeds", "data": _encode_tensor(t_a)},
{"type": "text", "text": " world "},
{"type": "prompt_embeds", "data": _encode_tensor(t_b)},
{"type": "text", "text": "!"},
],
}
]
conv, mm_data, _ = await _maybe_await(
parse_fn, messages, mc, content_format="openai"
)
tensors = list(mm_data["prompt_embeds"])
assert len(tensors) == NUM_TENSORS
# Tokenize: each prompt_embeds part becomes 1 placeholder token.
# `return_dict=False` to get a flat `list[int]` on transformers v5
# (where the default flipped to True and yields a `BatchEncoding` dict).
token_ids = tokenizer.apply_chat_template(conv, tokenize=True, return_dict=False)
assert sum(t == tid for t in token_ids) == NUM_TENSORS
# Expand, locate, and build.
mm_updates = _build_prompt_embeds_updates(tensors, tid)
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
assert len(expanded) == len(token_ids) + LEN_A + LEN_B - NUM_TENSORS
positions = _build_prompt_embeds_positions(expanded, len(tensors), mm_updates)
assert positions[0][1] == LEN_A
assert positions[1][1] == LEN_B
embeds, mask = _build_mixed_prompt_embeds(expanded, tensors, positions)
assert embeds.shape == (len(expanded), _MOCK_HIDDEN_SIZE)
assert mask.count(False) == LEN_A + LEN_B
assert torch.equal(embeds[positions[0][0] : positions[0][0] + LEN_A], t_a)
assert torch.equal(embeds[positions[1][0] : positions[1][0] + LEN_B], t_b)
@pytest.mark.asyncio
async def test_end_to_end_multi_message_conversation(tokenizer, parse_fn):
"""Full pipeline with prompt_embeds spread across system + user messages,
verifying ordering and positioning in the final token stream."""
tokenizer.chat_template = _SIMPLE_CHAT_TEMPLATE
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
LEN_SYS, LEN_USR = 4, 3
t_sys = torch.randn(LEN_SYS, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
t_usr = torch.randn(LEN_USR, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
NUM_TENSORS = 2 # t_sys and t_usr.
mc = _make_mock_model_config()
messages = [
{
"role": "system",
"content": [
{"type": "text", "text": "You are helpful."},
{"type": "prompt_embeds", "data": _encode_tensor(t_sys)},
],
},
{
"role": "user",
"content": [
{"type": "prompt_embeds", "data": _encode_tensor(t_usr)},
{"type": "text", "text": "Summarize."},
],
},
]
conv, mm_data, _ = await _maybe_await(
parse_fn, messages, mc, content_format="openai"
)
tensors = list(mm_data["prompt_embeds"])
assert len(tensors) == NUM_TENSORS
# Tokenize, expand, locate, and build.
# `return_dict=False` to get a flat `list[int]` on transformers v5
# (where the default flipped to True and yields a `BatchEncoding` dict).
token_ids = tokenizer.apply_chat_template(conv, tokenize=True, return_dict=False)
mm_updates = _build_prompt_embeds_updates(tensors, tid)
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
positions = _build_prompt_embeds_positions(expanded, len(tensors), mm_updates)
assert positions[0][1] == LEN_SYS
assert positions[1][1] == LEN_USR
# System embed must appear before user embed in the token stream.
assert positions[0][0] < positions[1][0]
embeds, mask = _build_mixed_prompt_embeds(expanded, tensors, positions)
assert embeds.shape == (len(expanded), _MOCK_HIDDEN_SIZE)
assert mask.count(False) == LEN_SYS + LEN_USR
assert torch.equal(embeds[positions[0][0] : positions[0][0] + LEN_SYS], t_sys)
assert torch.equal(embeds[positions[1][0] : positions[1][0] + LEN_USR], t_usr)
def test_truncation_keeps_the_mixed_mask_aligned_with_the_prompt():
"""`prompt_is_token_ids` must be truncated with `prompt_token_ids`.
The mask is positional: the engine pairs it element-wise with the prompt
tokens to decide which positions read a real token id and which read a row
of `prompt_embeds`. Truncating the prompt without truncating the mask
leaves the two describing different positions, so embed placeholders stop
being recognised as such.
"""
hidden_size = 8
placeholder_token_id = 0
num_head, embed_len, num_tail = 10, 5, 5
token_ids = (
list(range(1, num_head + 1))
+ [placeholder_token_id] * embed_len
+ list(range(1, num_tail + 1))
)
tensors = [torch.randn(embed_len, hidden_size)]
positions = [(num_head, embed_len)]
embeds, mask = _build_mixed_prompt_embeds(token_ids, tensors, positions)
prompt = {
"prompt_token_ids": token_ids,
"prompt_embeds": embeds,
"prompt_is_token_ids": mask,
}
keep = 8
tok_params = TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=keep,
truncation_side="left",
)
result = tok_params.apply_post_tokenization(None, prompt)
# Keeping the last 8 of 20 positions starts inside the embed span, so the
# first 3 surviving positions are embed rows and the rest are real tokens.
# Comparing contents rather than lengths is what pins the alignment: a
# mask reduced from the wrong side has the right length and still
# describes the wrong positions.
assert result["prompt_is_token_ids"] == [False] * 3 + [True] * num_tail
assert result["prompt_token_ids"] == token_ids[-keep:]
assert torch.equal(result["prompt_embeds"], embeds[-keep:])
@@ -0,0 +1,150 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from vllm.config import ModelConfig
from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY
from vllm.renderers.chat_utils import (
ChatCompletionMessageParam,
parse_chat_messages,
parse_chat_messages_async,
)
MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
@pytest.fixture(scope="function")
def mistral_model_config():
return ModelConfig(
MISTRAL_MODEL_ID,
runner="generate",
limit_mm_per_prompt={
"image": 2,
},
)
def test_parse_chat_messages_empty_system(
mistral_model_config,
):
# Test string format
conversation, _, _ = parse_chat_messages(
[
{"role": "system", "content": ""},
{
"role": "user",
"content": [{"type": "text", "text": "Who are you?"}],
},
],
mistral_model_config,
content_format="string",
)
assert conversation == [
{"role": "system", "content": ""},
{"role": "user", "content": "Who are you?"},
]
# Test openai format
conversation, _, _ = parse_chat_messages(
[
{"role": "system", "content": ""},
{
"role": "user",
"content": [{"type": "text", "text": "Who are you?"}],
},
],
mistral_model_config,
content_format="openai",
)
assert conversation == [
{"role": "system", "content": [{"type": "text", "text": ""}]},
{"role": "user", "content": [{"type": "text", "text": "Who are you?"}]},
]
@pytest.mark.asyncio
async def test_text_only_chat_does_not_initialize_media_connector(
mistral_model_config,
monkeypatch,
):
load_connector = MagicMock()
monkeypatch.setattr(MEDIA_CONNECTOR_REGISTRY, "load", load_connector)
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Who are you?"}
]
parse_chat_messages(
messages,
mistral_model_config,
content_format="string",
)
await parse_chat_messages_async(
messages,
mistral_model_config,
content_format="string",
)
load_connector.assert_not_called()
def test_parse_chat_messages_include_thinking_chunk(mistral_model_config):
messages = [
{
"role": "system",
"content": [
{"type": "text", "text": "You are a helpful assistant."},
{
"type": "thinking",
"closed": True,
"thinking": "Only return the answer when you are confident.",
},
],
},
{"role": "user", "content": "What is 2+2?"},
{
"role": "assistant",
"content": [
{"type": "text", "text": "Let me think about it."},
{"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
{
"type": "text",
"text": "The answer is 4.",
},
],
},
]
conversation_with_thinking, _, _ = parse_chat_messages(
messages,
mistral_model_config,
content_format="openai",
)
expected_conversation = [
{
"role": "system",
"content": [
{"type": "text", "text": "You are a helpful assistant."},
{
"type": "text",
"text": "Only return the answer when you are confident.",
},
],
},
{
"role": "user",
"content": [{"type": "text", "text": "What is 2+2?"}],
},
{
"role": "assistant",
"content": [
{"type": "text", "text": "Let me think about it."},
{"type": "text", "text": "2+2 = 4"},
{"type": "text", "text": "The answer is 4."},
],
},
]
assert conversation_with_thinking == expected_conversation
@@ -0,0 +1,148 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.renderers.chat_utils import ConversationMessage
from vllm.renderers.chat_utils.chat_message import _postprocess_messages
def _assistant_tool_call(arguments, name="write"):
return [
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_0",
"type": "function",
"function": {"name": name, "arguments": arguments},
}
],
}
]
def _assistant_tool_calls(calls):
return [
{
"role": "assistant",
"content": "",
"tool_calls": calls,
}
]
def test_tool_call_arguments_dict_passthrough(caplog):
messages = _assistant_tool_call({"a": 1})
_postprocess_messages(messages)
args = messages[0]["tool_calls"][0]["function"]["arguments"]
assert args == {"a": 1}
assert not caplog.records
def test_tool_call_arguments_valid_object_string(caplog):
messages = _assistant_tool_call('{"a": 1}')
_postprocess_messages(messages)
args = messages[0]["tool_calls"][0]["function"]["arguments"]
assert args == {"a": 1}
assert not caplog.records
def test_tool_call_arguments_malformed_json_small(caplog):
bad = '{"cmd": "mkdir -p /tmp/mmadtest && cat > /tmp/mmadtest'
messages = _assistant_tool_call(bad, name="exec")
_postprocess_messages(messages)
args = messages[0]["tool_calls"][0]["function"]["arguments"]
assert args == {}
assert len(caplog.records) == 1
assert "coercing to an empty object" in caplog.records[0].message
def test_tool_call_arguments_malformed_json_large(caplog):
bad = '{"filepath": "src/main.py", "contents": "' + ("x" * 18000)
messages = _assistant_tool_call(bad, name="write")
_postprocess_messages(messages)
args = messages[0]["tool_calls"][0]["function"]["arguments"]
assert args == {}
assert len(caplog.records) == 1
assert "coercing to an empty object" in caplog.records[0].message
assert f"{len(bad)} chars" in caplog.records[0].message
@pytest.mark.parametrize("non_obj", ["[]", "42", "true", '"hello"', "null"])
def test_tool_call_arguments_valid_json_non_object(caplog, non_obj):
messages = _assistant_tool_call(non_obj, name="bad_tool")
_postprocess_messages(messages)
args = messages[0]["tool_calls"][0]["function"]["arguments"]
assert args == {}
if non_obj == "null":
# null parses to None, so no warning is emitted according to requirements
assert not caplog.records
else:
assert len(caplog.records) == 1
assert "not a JSON object" in caplog.records[0].message
@pytest.mark.parametrize("missing", [None, ""])
def test_tool_call_arguments_missing_or_empty(caplog, missing):
messages = _assistant_tool_call(missing)
if missing is None:
del messages[0]["tool_calls"][0]["function"]["arguments"]
_postprocess_messages(messages)
args = messages[0]["tool_calls"][0]["function"]["arguments"]
assert args == {}
assert not caplog.records
def test_tool_call_arguments_multiple_independent(caplog):
calls = [
{
"id": "call_0",
"type": "function",
"function": {"name": "good", "arguments": '{"a": 1}'},
},
{
"id": "call_1",
"type": "function",
"function": {"name": "bad", "arguments": '{"cmd": "'},
},
]
messages = _assistant_tool_calls(calls)
_postprocess_messages(messages)
tool_calls = messages[0]["tool_calls"]
assert tool_calls[0]["function"]["arguments"] == {"a": 1}
assert tool_calls[1]["function"]["arguments"] == {}
assert len(caplog.records) == 1
assert "coercing to an empty object" in caplog.records[0].message
def test_postprocess_messages_null_arguments_string():
"""arguments="null" must not reach the chat template as Python None.
json.loads("null") returns None, which causes Jinja2 templates that call
tc.arguments.items() to raise 'None' has no attribute 'items'.
The function should coerce it to {} instead.
"""
messages: list[ConversationMessage] = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "get_current_time", "arguments": "null"},
}
],
}
]
_postprocess_messages(messages)
tool_calls = messages[0]["tool_calls"]
assert tool_calls is not None
assert tool_calls[0]["function"]["arguments"] == {}
+497
View File
@@ -0,0 +1,497 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.renderers.chat_utils import parse_chat_messages, parse_chat_messages_async
from vllm.utils.serial_utils import tensor2base64
from .utils import (
assert_mm_data_inputs,
assert_mm_uuids,
)
QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
@pytest.fixture(scope="function")
def qwen25omni_model_config_image_embeds():
return ModelConfig(
QWEN25OMNI_MODEL_ID,
runner="generate",
limit_mm_per_prompt={"image": 2},
enable_mm_embeds=True,
)
@pytest.fixture(scope="function")
def qwen25omni_model_config_video_embeds():
return ModelConfig(
QWEN25OMNI_MODEL_ID,
runner="generate",
limit_mm_per_prompt={"video": 2},
enable_mm_embeds=True,
)
@pytest.fixture(scope="function")
def qwen25omni_model_config_mm_interleaved():
return ModelConfig(
QWEN25OMNI_MODEL_ID,
runner="generate",
interleave_mm_strings=True,
limit_mm_per_prompt={
"image": 2,
"audio": 1,
"video": 1,
},
)
def test_parse_chat_messages_empty_video_embeds_with_uuid(
qwen25omni_model_config_video_embeds,
):
"""Test video_embeds with UUID (no actual embeds data)."""
uuid = "test-video-uuid-123"
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this video"},
{"type": "video_embeds", "video_embeds": None, "uuid": uuid},
],
}
],
qwen25omni_model_config_video_embeds,
content_format="string",
)
# Should have video in mm_data as None (UUID provided)
assert mm_data is not None
assert "video" in mm_data
assert isinstance(mm_data["video"], list)
assert len(mm_data["video"]) == 1
assert mm_data["video"][0] is None
# UUID should be recorded
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[uuid])
def test_parse_chat_messages_video_embeds_with_string(
qwen25omni_model_config_video_embeds,
):
"""Test video_embeds with base64 string embedding data."""
hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()
video_embedding = torch.randn(1, 128, hidden_size)
base64_video_embedding = tensor2base64(video_embedding)
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this video"},
{
"type": "video_embeds",
"video_embeds": base64_video_embedding,
},
],
}
],
qwen25omni_model_config_video_embeds,
content_format="string",
)
# Should have video embedding in mm_data (single tensor, not a list)
assert mm_data is not None
assert "video" in mm_data
assert isinstance(mm_data["video"], torch.Tensor)
assert mm_data["video"].shape == video_embedding.shape
# No UUID provided
assert mm_uuids is not None
assert "video" in mm_uuids
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
@pytest.mark.asyncio
async def test_parse_chat_messages_video_embeds_async(
qwen25omni_model_config_video_embeds,
):
"""Test video_embeds with async futures."""
hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()
video_embedding = torch.randn(1, 128, hidden_size)
base64_video_embedding = tensor2base64(video_embedding)
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
[
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this video"},
{
"type": "video_embeds",
"video_embeds": base64_video_embedding,
},
],
}
],
qwen25omni_model_config_video_embeds,
content_format="string",
)
# Should have video embedding in mm_data (single tensor, not a list)
assert mm_data is not None
assert "video" in mm_data
assert isinstance(mm_data["video"], torch.Tensor)
assert mm_data["video"].shape == video_embedding.shape
# No UUID provided
assert mm_uuids is not None
assert "video" in mm_uuids
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
def test_parse_chat_messages_video_and_video_embeds_mixed(
qwen25omni_model_config_video_embeds,
video_url,
):
"""Test that mixing raw video and video_embeds is rejected."""
with pytest.raises(VLLMValidationError, match="Mixing raw video"):
parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "video_url", "video_url": {"url": video_url}},
{"type": "video_embeds", "video_embeds": None, "uuid": "x"},
],
}
],
qwen25omni_model_config_video_embeds,
content_format="string",
)
@pytest.mark.parametrize("json_metadata", [False, True])
def test_parse_chat_messages_multiple_dict_image_embeds(
qwen25omni_model_config_image_embeds,
json_metadata,
):
"""Test that multiple dictionaries for image_embeds is handled without errors."""
# Create two sample image embedding tensors
batch_size = 2
hidden_size = qwen25omni_model_config_image_embeds.get_inputs_embeds_size()
image_embeds = torch.randn(batch_size * 220, hidden_size)
image_grid_thw = torch.tensor([[1, 22, 40] for _ in range(batch_size)])
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{
"type": "image_embeds",
"image_embeds": {
"image_embeds": tensor2base64(embeds),
"image_grid_thw": grid_thw.tolist()
if json_metadata
else tensor2base64(grid_thw),
},
}
for embeds, grid_thw in zip(
image_embeds.chunk(batch_size), image_grid_thw
)
]
+ [
{"type": "text", "text": "Describe these two images."},
],
}
],
qwen25omni_model_config_image_embeds,
content_format="string",
)
# Verify conversation structure
assert conversation == [
{
"role": "user",
"content": "<|vision_start|><|IMAGE|><|vision_end|>\n"
"<|vision_start|><|IMAGE|><|vision_end|>\nDescribe these two images.",
}
]
# Verify mm_data contains a dictionary of multi-embeddings
assert mm_data is not None
assert "image" in mm_data
assert isinstance(mm_data["image"], dict)
assert len(mm_data["image"]) == batch_size
# Verify each embedding has the correct shape
assert isinstance(mm_data["image"]["image_embeds"], torch.Tensor)
assert mm_data["image"]["image_embeds"].shape == image_embeds.shape
assert isinstance(mm_data["image"]["image_grid_thw"], torch.Tensor)
assert mm_data["image"]["image_grid_thw"].shape == image_grid_thw.shape
# Verify UUIDs (None since we didn't provide any)
assert_mm_uuids(mm_uuids, batch_size, expected_uuids=[None, None])
def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
qwen25omni_model_config_mm_interleaved,
image_url,
video_url,
audio_url,
):
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "text", "text": "What's on this image?"},
{"type": "image_url", "image_url": {"url": image_url}},
{"type": "text", "text": "Now listen to this audio"},
{"type": "audio_url", "audio_url": {"url": audio_url}},
],
},
{"role": "assistant", "content": "Some stuff."},
{
"role": "user",
"content": [
{"type": "text", "text": "What's on this image?"},
{"type": "image_url", "image_url": {"url": image_url}},
{"type": "text", "text": "And what's in the video?"},
{"type": "video_url", "video_url": {"url": video_url}},
],
},
],
qwen25omni_model_config_mm_interleaved,
content_format="string",
)
assert conversation == [
{
"role": "user",
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
"\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
},
{"role": "assistant", "content": "Some stuff."},
{
"role": "user",
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
"\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
},
]
assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
assert_mm_uuids(mm_uuids, 2, modality="image", expected_uuids=[None, None])
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interleave(
qwen25omni_model_config_mm_interleaved,
image_url,
video_url,
audio_url,
):
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "text", "text": "What's on this image?"},
{
"type": "image_url",
"image_url": {"url": image_url},
"uuid": "image_123",
},
{"type": "text", "text": "Now listen to this audio"},
{
"type": "audio_url",
"audio_url": {"url": audio_url},
"uuid": "audio_123",
},
],
},
{"role": "assistant", "content": "Some stuff."},
{
"role": "user",
"content": [
{"type": "text", "text": "What's on this image?"},
{
"type": "image_url",
"image_url": {"url": image_url},
"uuid": "image_123",
},
{"type": "text", "text": "And what's in the video?"},
{
"type": "video_url",
"video_url": {"url": video_url},
"uuid": "video_123",
},
],
},
],
qwen25omni_model_config_mm_interleaved,
content_format="string",
)
assert conversation == [
{
"role": "user",
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
"\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
},
{"role": "assistant", "content": "Some stuff."},
{
"role": "user",
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
"\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
},
]
assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
assert_mm_uuids(
mm_uuids, 2, modality="image", expected_uuids=["image_123", "image_123"]
)
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=["audio_123"])
def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_messages_interleave( # noqa: E501
qwen25omni_model_config_mm_interleaved,
image_url,
video_url,
audio_url,
):
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "text", "text": "What's on this image?"},
{
"type": "image_url",
"image_url": None,
"uuid": "image_123",
},
{"type": "text", "text": "Now listen to this audio"},
{
"type": "audio_url",
"audio_url": None,
"uuid": "audio_123",
},
],
},
{"role": "assistant", "content": "Some stuff."},
{
"role": "user",
"content": [
{"type": "text", "text": "What's on this image?"},
{
"type": "image_url",
"image_url": None,
"uuid": "image_123",
},
{"type": "text", "text": "And what's in the video?"},
{
"type": "video_url",
"video_url": None,
"uuid": "video_123",
},
],
},
],
qwen25omni_model_config_mm_interleaved,
content_format="string",
)
assert conversation == [
{
"role": "user",
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
"\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
},
{"role": "assistant", "content": "Some stuff."},
{
"role": "user",
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
"\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
},
]
assert_mm_data_inputs(
mm_data,
{"image": 2, "video": 1, "audio": 1},
skipped_media_indices={"image": [0, 1], "video": [0], "audio": [0]},
)
assert_mm_uuids(
mm_uuids, 2, modality="image", expected_uuids=["image_123", "image_123"]
)
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=["audio_123"])
def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_messages_interleave( # noqa: E501
qwen25omni_model_config_mm_interleaved,
image_url,
video_url,
audio_url,
):
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "text", "text": "What's on this image?"},
{
"type": "image_url",
"image_url": {"url": image_url},
"uuid": "image_123",
},
{"type": "text", "text": "Now listen to this audio"},
{"type": "audio_url", "audio_url": {"url": audio_url}},
],
},
{"role": "assistant", "content": "Some stuff."},
{
"role": "user",
"content": [
{"type": "text", "text": "What's on this image?"},
{"type": "image_url", "image_url": {"url": image_url}},
{"type": "text", "text": "And what's in the video?"},
{
"type": "video_url",
"video_url": {"url": video_url},
"uuid": "video_123",
},
],
},
],
qwen25omni_model_config_mm_interleaved,
content_format="string",
)
assert conversation == [
{
"role": "user",
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
"\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
},
{"role": "assistant", "content": "Some stuff."},
{
"role": "user",
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
"\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
},
]
assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
assert_mm_uuids(mm_uuids, 2, modality="image", expected_uuids=["image_123", None])
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
@@ -0,0 +1,546 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.config import ModelConfig
from vllm.inputs import MultiModalDataDict
from vllm.renderers.chat_utils import (
ChatCompletionMessageParam,
parse_chat_messages,
parse_chat_messages_async,
)
from .utils import assert_mm_uuids
KIMI_K2_5_MODEL_ID = "moonshotai/Kimi-K2.5"
@pytest.fixture(scope="function")
def kimi_k2_5_model_config():
return ModelConfig(
KIMI_K2_5_MODEL_ID,
runner="generate",
trust_remote_code=True,
limit_mm_per_prompt={
"image": 2,
},
)
def _assert_mm_data_is_vision_chunk_input(
mm_data: MultiModalDataDict | None,
vision_chunk_count: int,
) -> None:
assert mm_data is not None
assert set(mm_data.keys()) == {"vision_chunk"}
vision_chunk_data = mm_data.get("vision_chunk")
assert vision_chunk_data is not None
assert (
isinstance(vision_chunk_data, list)
and len(vision_chunk_data) == vision_chunk_count
)
def test_parse_chat_messages_image_vision_chunk(
kimi_k2_5_model_config,
image_url,
):
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this image."},
{
"type": "image_url",
"image_url": {"url": image_url},
},
],
}
]
conversation, mm_data, mm_uuids = parse_chat_messages(
messages,
kimi_k2_5_model_config,
content_format="string",
)
placeholder = "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
expected_conversation = [
{
"role": "user",
"content": f"{placeholder}\nAnalyze this image.",
}
]
assert conversation == expected_conversation
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
assert_mm_uuids(mm_uuids, 1, expected_uuids=[None], modality="vision_chunk")
def test_parse_chat_messages_video_vision_chunk(
kimi_k2_5_model_config,
video_url,
):
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this video."},
{
"type": "video_url",
"video_url": {"url": video_url},
},
],
}
]
conversation, mm_data, mm_uuids = parse_chat_messages(
messages,
kimi_k2_5_model_config,
content_format="string",
)
placeholder = "<|kimi_k25_video_placeholder|>"
expected_conversation = [
{
"role": "user",
"content": f"{placeholder}\nAnalyze this video.",
}
]
assert conversation == expected_conversation
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
assert_mm_uuids(mm_uuids, 1, expected_uuids=[None], modality="vision_chunk")
def test_parse_chat_messages_image_vision_chunk_with_uuid(
kimi_k2_5_model_config,
image_url,
):
image_uuid = "image_123"
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this image."},
{
"type": "image_url",
"image_url": {"url": image_url},
"uuid": image_uuid,
},
],
}
]
conversation, mm_data, mm_uuids = parse_chat_messages(
messages,
kimi_k2_5_model_config,
content_format="string",
)
placeholder = "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
expected_conversation = [
{
"role": "user",
"content": f"{placeholder}\nAnalyze this image.",
}
]
assert conversation == expected_conversation
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid], modality="vision_chunk")
def test_parse_chat_messages_video_vision_chunk_with_uuid(
kimi_k2_5_model_config,
video_url,
):
video_uuid = "video_456"
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this video."},
{
"type": "video_url",
"video_url": {"url": video_url},
"uuid": video_uuid,
},
],
}
]
conversation, mm_data, mm_uuids = parse_chat_messages(
messages,
kimi_k2_5_model_config,
content_format="string",
)
placeholder = "<|kimi_k25_video_placeholder|>"
expected_conversation = [
{
"role": "user",
"content": f"{placeholder}\nAnalyze this video.",
}
]
assert conversation == expected_conversation
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
assert_mm_uuids(mm_uuids, 1, expected_uuids=[video_uuid], modality="vision_chunk")
def test_parse_chat_messages_mixed_vision_chunk(
kimi_k2_5_model_config,
image_url,
video_url,
):
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this image and video."},
{
"type": "image_url",
"image_url": {"url": image_url},
},
{
"type": "video_url",
"video_url": {"url": video_url},
},
],
}
]
conversation, mm_data, mm_uuids = parse_chat_messages(
messages,
kimi_k2_5_model_config,
content_format="string",
)
image_placeholder = (
"<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
)
video_placeholder = "<|kimi_k25_video_placeholder|>"
expected_conversation = [
{
"role": "user",
"content": (
f"{image_placeholder}\n{video_placeholder}\n"
"Analyze this image and video."
),
}
]
assert conversation == expected_conversation
_assert_mm_data_is_vision_chunk_input(mm_data, 2)
assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None], modality="vision_chunk")
def test_parse_chat_messages_mixed_vision_chunk_with_uuid(
kimi_k2_5_model_config,
image_url,
video_url,
):
image_uuid = "image_123"
video_uuid = "video_456"
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this image and video."},
{
"type": "image_url",
"image_url": {"url": image_url},
"uuid": image_uuid,
},
{
"type": "video_url",
"video_url": {"url": video_url},
"uuid": video_uuid,
},
],
}
]
conversation, mm_data, mm_uuids = parse_chat_messages(
messages,
kimi_k2_5_model_config,
content_format="string",
)
image_placeholder = (
"<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
)
video_placeholder = "<|kimi_k25_video_placeholder|>"
expected_conversation = [
{
"role": "user",
"content": (
f"{image_placeholder}\n{video_placeholder}\n"
"Analyze this image and video."
),
}
]
assert conversation == expected_conversation
_assert_mm_data_is_vision_chunk_input(mm_data, 2)
assert_mm_uuids(
mm_uuids, 2, expected_uuids=[image_uuid, video_uuid], modality="vision_chunk"
)
@pytest.mark.asyncio
async def test_parse_chat_messages_mixed_vision_chunk_async(
kimi_k2_5_model_config,
image_url,
video_url,
):
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this image and video."},
{
"type": "image_url",
"image_url": {"url": image_url},
},
{
"type": "video_url",
"video_url": {"url": video_url},
},
],
}
]
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
messages,
kimi_k2_5_model_config,
content_format="string",
)
image_placeholder = (
"<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
)
video_placeholder = "<|kimi_k25_video_placeholder|>"
expected_conversation = [
{
"role": "user",
"content": (
f"{image_placeholder}\n{video_placeholder}\n"
"Analyze this image and video."
),
}
]
assert conversation == expected_conversation
_assert_mm_data_is_vision_chunk_input(mm_data, 2)
assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None], modality="vision_chunk")
@pytest.mark.asyncio
async def test_parse_chat_messages_mixed_vision_chunk_with_uuid_async(
kimi_k2_5_model_config,
image_url,
video_url,
):
image_uuid = "image_123"
video_uuid = "video_456"
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this image and video."},
{
"type": "image_url",
"image_url": {"url": image_url},
"uuid": image_uuid,
},
{
"type": "video_url",
"video_url": {"url": video_url},
"uuid": video_uuid,
},
],
}
]
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
messages,
kimi_k2_5_model_config,
content_format="string",
)
image_placeholder = (
"<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
)
video_placeholder = "<|kimi_k25_video_placeholder|>"
expected_conversation = [
{
"role": "user",
"content": (
f"{image_placeholder}\n{video_placeholder}\n"
"Analyze this image and video."
),
}
]
assert conversation == expected_conversation
_assert_mm_data_is_vision_chunk_input(mm_data, 2)
assert_mm_uuids(
mm_uuids, 2, expected_uuids=[image_uuid, video_uuid], modality="vision_chunk"
)
@pytest.mark.asyncio
async def test_parse_chat_messages_image_vision_chunk_async(
kimi_k2_5_model_config,
image_url,
):
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this image."},
{
"type": "image_url",
"image_url": {"url": image_url},
},
],
}
]
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
messages,
kimi_k2_5_model_config,
content_format="string",
)
placeholder = "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
expected_conversation = [
{
"role": "user",
"content": f"{placeholder}\nAnalyze this image.",
}
]
assert conversation == expected_conversation
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
assert_mm_uuids(mm_uuids, 1, expected_uuids=[None], modality="vision_chunk")
@pytest.mark.asyncio
async def test_parse_chat_messages_video_vision_chunk_async(
kimi_k2_5_model_config,
video_url,
):
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this video."},
{
"type": "video_url",
"video_url": {"url": video_url},
},
],
}
]
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
messages,
kimi_k2_5_model_config,
content_format="string",
)
placeholder = "<|kimi_k25_video_placeholder|>"
expected_conversation = [
{
"role": "user",
"content": f"{placeholder}\nAnalyze this video.",
}
]
assert conversation == expected_conversation
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
assert_mm_uuids(mm_uuids, 1, expected_uuids=[None], modality="vision_chunk")
@pytest.mark.asyncio
async def test_parse_chat_messages_image_vision_chunk_with_uuid_async(
kimi_k2_5_model_config,
image_url,
):
image_uuid = "image_123"
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this image."},
{
"type": "image_url",
"image_url": {"url": image_url},
"uuid": image_uuid,
},
],
}
]
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
messages,
kimi_k2_5_model_config,
content_format="string",
)
placeholder = "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
expected_conversation = [
{
"role": "user",
"content": f"{placeholder}\nAnalyze this image.",
}
]
assert conversation == expected_conversation
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid], modality="vision_chunk")
@pytest.mark.asyncio
async def test_parse_chat_messages_video_vision_chunk_with_uuid_async(
kimi_k2_5_model_config,
video_url,
):
video_uuid = "video_456"
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this video."},
{
"type": "video_url",
"video_url": {"url": video_url},
"uuid": video_uuid,
},
],
}
]
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
messages,
kimi_k2_5_model_config,
content_format="string",
)
placeholder = "<|kimi_k25_video_placeholder|>"
expected_conversation = [
{
"role": "user",
"content": f"{placeholder}\nAnalyze this video.",
}
]
assert conversation == expected_conversation
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
assert_mm_uuids(mm_uuids, 1, expected_uuids=[video_uuid], modality="vision_chunk")
+66
View File
@@ -0,0 +1,66 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from typing import Literal
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
ModalityType = Literal["image", "video", "audio"]
MultiModalDataCounts = Mapping[ModalityType, int]
def assert_mm_uuids(
mm_uuids: MultiModalUUIDDict | None,
media_count: int,
expected_uuids: list[str | None],
modality: str = "image",
) -> None:
if len(expected_uuids) > 0:
assert mm_uuids is not None
assert modality in mm_uuids
image_uuids = mm_uuids.get(modality)
assert image_uuids is not None
assert isinstance(image_uuids, list) and len(image_uuids) == media_count
assert image_uuids == expected_uuids
else:
assert mm_uuids is None
def assert_mm_data_inputs(
mm_data: MultiModalDataDict | None,
data_count: MultiModalDataCounts,
skipped_media_indices: dict[str, list] | None = None, # modality -> list[int]
) -> None:
assert mm_data is not None
assert set(data_count.keys()) == (set(mm_data.keys()))
for modality, n in data_count.items():
modality_data = mm_data.get(modality)
assert modality_data is not None
assert isinstance(modality_data, list) and len(modality_data) == n
if skipped_media_indices is not None:
skipped_media_indices_for_modality = skipped_media_indices.get(modality)
assert skipped_media_indices_for_modality is not None
for i in skipped_media_indices_for_modality:
assert modality_data[i] is None
def assert_mm_data_is_image_input(
mm_data: MultiModalDataDict | None,
image_count: int,
skipped_image_indices: list | None = None,
) -> None:
assert mm_data is not None
assert set(mm_data.keys()) == {"image"}
image_data = mm_data.get("image")
assert image_data is not None
assert isinstance(image_data, list) and len(image_data) == image_count
if skipped_image_indices is not None:
for i in skipped_image_indices:
assert image_data[i] is None