[Test] Split the chat_utils tests. (#61765)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.multimodal.utils import encode_audio_url, encode_image_url, encode_video_url
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def image_url():
|
||||
image = ImageAsset("cherry_blossom")
|
||||
return encode_image_url(image.pil_image)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def video_url():
|
||||
video = VideoAsset("baby_reading", 1)
|
||||
return encode_video_url(video.np_ndarrays)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def audio_url():
|
||||
audio = AudioAsset("mary_had_lamb")
|
||||
return encode_audio_url(*audio.audio_and_sample_rate)
|
||||
@@ -0,0 +1,51 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.renderers.chat_utils import ConversationMessage, parse_chat_message_content
|
||||
|
||||
|
||||
def _parse_one(message) -> ConversationMessage:
|
||||
# Plain-text content only reads these two tracker attributes.
|
||||
tracker = MagicMock()
|
||||
parser = tracker.create_parser.return_value
|
||||
parser.model_config.enable_prompt_embeds = False
|
||||
parser.mm_placeholder_storage.return_value = {}
|
||||
result = parse_chat_message_content(
|
||||
message, tracker, content_format="string", interleave_strings=False
|
||||
)
|
||||
assert len(result) == 1
|
||||
return result[0]
|
||||
|
||||
|
||||
def test_assistant_reasoning_is_mirrored_to_reasoning_content():
|
||||
"""``reasoning`` is mirrored to ``reasoning_content`` for legacy templates."""
|
||||
conv_msg = _parse_one(
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "4",
|
||||
"reasoning": "2+2 equals 4",
|
||||
}
|
||||
)
|
||||
|
||||
assert conv_msg.get("reasoning") == "2+2 equals 4"
|
||||
assert conv_msg.get("reasoning_content") == "2+2 equals 4"
|
||||
|
||||
|
||||
def test_assistant_without_reasoning_sets_neither_field():
|
||||
"""No reasoning in, no reasoning keys out: no empty thinking block in templates."""
|
||||
conv_msg = _parse_one({"role": "assistant", "content": "Hi"})
|
||||
|
||||
assert "reasoning" not in conv_msg
|
||||
assert "reasoning_content" not in conv_msg
|
||||
|
||||
|
||||
def test_non_assistant_reasoning_is_not_mirrored():
|
||||
"""Only assistant turns carry reasoning; a user turn must not."""
|
||||
conv_msg = _parse_one(
|
||||
{"role": "user", "content": "Hi", "reasoning": "should be ignored"}
|
||||
)
|
||||
|
||||
assert "reasoning" not in conv_msg
|
||||
assert "reasoning_content" not in conv_msg
|
||||
@@ -0,0 +1,224 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.renderers.chat_utils import parse_chat_messages, parse_chat_messages_async
|
||||
from vllm.utils.serial_utils import tensor2base64
|
||||
|
||||
from .utils import (
|
||||
assert_mm_data_inputs,
|
||||
assert_mm_uuids,
|
||||
)
|
||||
|
||||
QWEN2AUDIO_MODEL_ID = "Qwen/Qwen2-Audio-7B-Instruct"
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def qwen2_audio_model_config():
|
||||
return ModelConfig(
|
||||
QWEN2AUDIO_MODEL_ID,
|
||||
runner="generate",
|
||||
trust_remote_code=True,
|
||||
limit_mm_per_prompt={
|
||||
"audio": 1,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def audio_embeds_model_config():
|
||||
return ModelConfig(
|
||||
QWEN2AUDIO_MODEL_ID,
|
||||
runner="generate",
|
||||
trust_remote_code=True,
|
||||
limit_mm_per_prompt={
|
||||
"audio": 2,
|
||||
},
|
||||
enable_mm_embeds=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("input_audio", [None, {}, {"data": "", "format": "wav"}])
|
||||
def test_parse_chat_messages_single_empty_audio_with_uuid(
|
||||
qwen2_audio_model_config,
|
||||
input_audio,
|
||||
):
|
||||
audio_uuid = "abcd"
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": input_audio,
|
||||
"uuid": audio_uuid,
|
||||
},
|
||||
{"type": "text", "text": "What does the audio say?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen2_audio_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
assert conversation == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the "
|
||||
"audio say?",
|
||||
}
|
||||
]
|
||||
assert_mm_data_inputs(mm_data, {"audio": 1})
|
||||
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("input_audio", [None, {}, {"data": "", "format": "wav"}])
|
||||
async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
|
||||
qwen2_audio_model_config,
|
||||
input_audio,
|
||||
):
|
||||
audio_uuid = "abcd"
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": input_audio,
|
||||
"uuid": audio_uuid,
|
||||
},
|
||||
{"type": "text", "text": "What does the audio say?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen2_audio_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
assert conversation == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the "
|
||||
"audio say?",
|
||||
}
|
||||
]
|
||||
assert_mm_data_inputs(mm_data, {"audio": 1})
|
||||
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
|
||||
|
||||
|
||||
def test_parse_chat_messages_empty_audio_embeds_with_uuid(
|
||||
audio_embeds_model_config,
|
||||
):
|
||||
"""Test audio_embeds with UUID (no actual embeds data)."""
|
||||
uuid = "test-audio-uuid-123"
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this audio"},
|
||||
{"type": "audio_embeds", "audio_embeds": None, "uuid": uuid},
|
||||
],
|
||||
}
|
||||
],
|
||||
audio_embeds_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Should have audio in mm_data as None (UUID provided)
|
||||
assert mm_data is not None
|
||||
assert "audio" in mm_data
|
||||
assert isinstance(mm_data["audio"], list)
|
||||
assert len(mm_data["audio"]) == 1
|
||||
assert mm_data["audio"][0] is None
|
||||
|
||||
# UUID should be recorded
|
||||
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[uuid])
|
||||
|
||||
|
||||
def test_parse_chat_messages_audio_embeds_with_string(
|
||||
audio_embeds_model_config,
|
||||
):
|
||||
"""Test audio_embeds with base64 string embedding data."""
|
||||
import torch
|
||||
|
||||
# Create a sample audio embedding tensor
|
||||
hidden_size = audio_embeds_model_config.get_inputs_embeds_size()
|
||||
audio_embedding = torch.randn(1, 128, hidden_size)
|
||||
|
||||
# Encode it as base64
|
||||
base64_audio_embedding = tensor2base64(audio_embedding)
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this audio"},
|
||||
{
|
||||
"type": "audio_embeds",
|
||||
"audio_embeds": base64_audio_embedding,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
audio_embeds_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Should have audio embedding in mm_data (single tensor, not a list)
|
||||
assert mm_data is not None
|
||||
assert "audio" in mm_data
|
||||
assert isinstance(mm_data["audio"], torch.Tensor)
|
||||
assert mm_data["audio"].shape == audio_embedding.shape
|
||||
# No UUID provided
|
||||
assert mm_uuids is not None
|
||||
assert "audio" in mm_uuids
|
||||
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_chat_messages_audio_embeds_async(
|
||||
audio_embeds_model_config,
|
||||
):
|
||||
"""Test audio_embeds with async futures."""
|
||||
import torch
|
||||
|
||||
# Create a sample audio embedding tensor
|
||||
hidden_size = audio_embeds_model_config.get_inputs_embeds_size()
|
||||
audio_embedding = torch.randn(1, 128, hidden_size)
|
||||
|
||||
# Encode it as base64
|
||||
base64_audio_embedding = tensor2base64(audio_embedding)
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this audio"},
|
||||
{
|
||||
"type": "audio_embeds",
|
||||
"audio_embeds": base64_audio_embedding,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
audio_embeds_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Should have audio embedding in mm_data (single tensor, not a list)
|
||||
assert mm_data is not None
|
||||
assert "audio" in mm_data
|
||||
assert isinstance(mm_data["audio"], torch.Tensor)
|
||||
assert mm_data["audio"].shape == audio_embedding.shape
|
||||
# No UUID provided
|
||||
assert mm_uuids is not None
|
||||
assert "audio" in mm_uuids
|
||||
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
|
||||
@@ -0,0 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers.chat_utils import validate_chat_template
|
||||
|
||||
|
||||
def test_validate_chat_template_rejects_invalid_type():
|
||||
"""A non-str/Path chat_template is invalid user input."""
|
||||
with pytest.raises(
|
||||
VLLMValidationError, match="not a valid chat template type"
|
||||
) as exc_info:
|
||||
validate_chat_template(123)
|
||||
assert exc_info.value.parameter == "chat_template"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.renderers.chat_utils import AsyncMultiModalItemTracker
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_items_runs_modalities_concurrently_and_preserves_order():
|
||||
"""Media fetches overlap while modality and item order are preserved."""
|
||||
active_fetches = 0
|
||||
max_active_fetches = 0
|
||||
|
||||
async def _fetch(name: str, delay: float):
|
||||
nonlocal active_fetches, max_active_fetches
|
||||
active_fetches += 1
|
||||
max_active_fetches = max(max_active_fetches, active_fetches)
|
||||
try:
|
||||
await asyncio.sleep(delay)
|
||||
return name, f"{name}-uuid"
|
||||
finally:
|
||||
active_fetches -= 1
|
||||
|
||||
tracker = AsyncMultiModalItemTracker(MagicMock(is_multimodal_model=True))
|
||||
tracker.__dict__["mm_processor"] = MagicMock()
|
||||
tracker._items_by_modality["video"] = [
|
||||
lambda: _fetch("video-0", 0.02),
|
||||
lambda: _fetch("video-1", 0.01),
|
||||
]
|
||||
tracker._items_by_modality["image"] = [
|
||||
lambda: _fetch("image-0", 0.02),
|
||||
lambda: _fetch("image-1", 0.01),
|
||||
]
|
||||
tracker._items_by_modality["audio"] = [
|
||||
lambda: _fetch("audio-0", 0.02),
|
||||
lambda: _fetch("audio-1", 0.01),
|
||||
]
|
||||
|
||||
mm_data, mm_uuids = await tracker.resolve_items()
|
||||
|
||||
assert mm_data is not None
|
||||
assert mm_uuids is not None
|
||||
assert max_active_fetches == 6
|
||||
assert list(mm_data) == ["image", "audio", "video"]
|
||||
assert list(mm_uuids) == ["image", "audio", "video"]
|
||||
assert mm_data == {
|
||||
"image": ["image-0", "image-1"],
|
||||
"audio": ["audio-0", "audio-1"],
|
||||
"video": ["video-0", "video-1"],
|
||||
}
|
||||
assert mm_uuids == {
|
||||
"image": ["image-0-uuid", "image-1-uuid"],
|
||||
"audio": ["audio-0-uuid", "audio-1-uuid"],
|
||||
"video": ["video-0-uuid", "video-1-uuid"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_items_does_not_leak_tasks_on_partial_failure():
|
||||
"""Regression test: one failing media fetch must not abandon the other
|
||||
still-in-flight fetches across modality batches.
|
||||
|
||||
Before the fix, `resolve_items` gathered per-modality fetches with plain
|
||||
`asyncio.gather`, so the first exception propagated immediately while
|
||||
sibling fetches (real network/thread-pool work in production) kept
|
||||
running detached, with nothing left to await or cancel them.
|
||||
"""
|
||||
|
||||
async def _fetch(should_fail: bool, delay: float):
|
||||
if should_fail:
|
||||
await asyncio.sleep(0.01)
|
||||
raise ValueError("simulated fetch failure")
|
||||
await asyncio.sleep(delay)
|
||||
return ("decoded", None)
|
||||
|
||||
tracker = AsyncMultiModalItemTracker(MagicMock())
|
||||
tracker._items_by_modality["image"] = [
|
||||
lambda: _fetch(True, 0),
|
||||
lambda: _fetch(False, 0.2),
|
||||
lambda: _fetch(False, 0.2),
|
||||
]
|
||||
tracker._items_by_modality["audio"] = [lambda: _fetch(False, 0.2)]
|
||||
tracker._items_by_modality["video"] = [lambda: _fetch(False, 0.2)]
|
||||
|
||||
tasks_before = asyncio.all_tasks()
|
||||
with pytest.raises(ValueError, match="simulated fetch failure"):
|
||||
await tracker.resolve_items()
|
||||
|
||||
leaked_tasks = asyncio.all_tasks() - tasks_before
|
||||
assert not leaked_tasks, (
|
||||
f"resolve_items left {len(leaked_tasks)} task(s) running after "
|
||||
f"raising: {leaked_tasks}"
|
||||
)
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers.chat_utils.content_parser import _load_embeds_dict
|
||||
from vllm.renderers.chat_utils.item_tracker import _parse_metadata_array
|
||||
|
||||
|
||||
@pytest.mark.parametrize("values", [[1, 32, 48], [0.0, 0.12345678912345678]])
|
||||
def test_json_metadata_preserves_numeric_precision(values):
|
||||
tensor = _parse_metadata_array("grid", values, {"grid"})
|
||||
assert tensor.tolist() == values
|
||||
assert tensor.dtype == (
|
||||
torch.float64 if isinstance(values[0], float) else torch.int64
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key,values",
|
||||
[
|
||||
("image_embeds", [1, 2]),
|
||||
("grid", [True]),
|
||||
("grid", [float("nan")]),
|
||||
("grid", [float("inf")]),
|
||||
("grid", [[1, 2]]),
|
||||
("grid", [2**64]),
|
||||
],
|
||||
)
|
||||
def test_json_arrays_are_only_valid_numeric_metadata(key, values):
|
||||
with pytest.raises(VLLMValidationError):
|
||||
_parse_metadata_array(key, values, {"grid"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_json_metadata_bypasses_tensor_deserialization():
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
tensor = torch.ones(2, 3)
|
||||
fetch = AsyncMock(return_value=tensor)
|
||||
result = await _load_embeds_dict(
|
||||
{"image_embeds": "legacy-base64", "image_grid_thw": [1, 2, 2]}, fetch
|
||||
)
|
||||
fetch.assert_awaited_once_with("legacy-base64")
|
||||
assert result["image_embeds"] is tensor
|
||||
assert result["image_grid_thw"] == [1, 2, 2]
|
||||
@@ -0,0 +1,625 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Offline unit tests for `prompt_embeds` chat-completion content parts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import io
|
||||
from typing import Final
|
||||
from unittest import mock
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import regex as re
|
||||
import torch
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.renderers.chat_utils import parse_chat_messages, parse_chat_messages_async
|
||||
from vllm.renderers.chat_utils.chat_message import (
|
||||
_PROMPT_EMBEDS_MISSING_DATA_ERROR,
|
||||
_RESERVED_PLACEHOLDER_IN_TEXT_ERROR,
|
||||
MM_PARSER_MAP,
|
||||
)
|
||||
from vllm.renderers.chat_utils.content_parser import (
|
||||
_ENABLE_PROMPT_EMBEDS_ERROR,
|
||||
MODALITY_PLACEHOLDERS_MAP,
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
|
||||
)
|
||||
from vllm.renderers.hf import (
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR,
|
||||
_build_mixed_prompt_embeds,
|
||||
_build_prompt_embeds_positions,
|
||||
_build_prompt_embeds_updates,
|
||||
_ensure_prompt_embeds_placeholder_token,
|
||||
_expand_prompt_embeds_placeholders,
|
||||
)
|
||||
|
||||
# Cover distinct tokenizer families:
|
||||
# GPT2TokenizerFast (BPE, OpenAI-style)
|
||||
# Qwen2TokenizerFast (SentencePiece BPE variant)
|
||||
# BertTokenizerFast (WordPiece)
|
||||
TOKENIZER_IDS: Final[list[str]] = [
|
||||
"openai-community/gpt2",
|
||||
"Qwen/Qwen2.5-1.5B-Instruct",
|
||||
"bert-base-uncased",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(params=TOKENIZER_IDS, ids=TOKENIZER_IDS)
|
||||
def tokenizer(request):
|
||||
"""A fresh tokenizer instance per tokenizer family."""
|
||||
return AutoTokenizer.from_pretrained(request.param)
|
||||
|
||||
|
||||
# Minimal chat template that works with any tokenizer. Iterates
|
||||
# `message.content` as either a string or a list of dicts (openai format).
|
||||
_SIMPLE_CHAT_TEMPLATE: Final[str] = (
|
||||
"{% for m in messages %}"
|
||||
"{% if m['content'] is string %}{{m['content']}}"
|
||||
"{% else %}{% for p in m['content'] %}{{p['text']}}{% endfor %}"
|
||||
"{% endif %}\n{% endfor %}"
|
||||
)
|
||||
|
||||
|
||||
async def _maybe_await(fn, *args, **kwargs):
|
||||
"""Call *fn* and `await` the result if it's a coroutine."""
|
||||
result = fn(*args, **kwargs)
|
||||
if inspect.iscoroutine(result):
|
||||
result = await result
|
||||
return result
|
||||
|
||||
|
||||
# Parametrize over sync / async parse paths so every end-to-end test
|
||||
# exercises both.
|
||||
_PARSE_FUNCTIONS = [parse_chat_messages, parse_chat_messages_async]
|
||||
|
||||
|
||||
@pytest.fixture(params=_PARSE_FUNCTIONS, ids=["sync", "async"])
|
||||
def parse_fn(request):
|
||||
"""Either the sync or async `parse_chat_messages` callable."""
|
||||
return request.param
|
||||
|
||||
|
||||
def _encode_tensor(t: torch.Tensor) -> str:
|
||||
buf = io.BytesIO()
|
||||
torch.save(t, buf)
|
||||
return base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
|
||||
|
||||
_MOCK_HIDDEN_SIZE: Final[int] = 8
|
||||
_MOCK_DTYPE: Final[torch.dtype] = torch.float32
|
||||
|
||||
|
||||
def _make_mock_model_config(*, enable_prompt_embeds: bool = True) -> mock.MagicMock:
|
||||
mc = mock.MagicMock()
|
||||
mc.enable_prompt_embeds = enable_prompt_embeds
|
||||
mc.multimodal_config = None
|
||||
mc.allowed_local_media_path = None
|
||||
mc.allowed_media_domains = None
|
||||
# Test text-only code path in `MultiModalItemTracker.resolve_items`.
|
||||
mc.is_multimodal_model = False
|
||||
# `safe_load_prompt_embeds` pins each tensor to the model's hidden_size
|
||||
# and dtype, so the mock must return concrete values.
|
||||
mc.get_hidden_size.return_value = _MOCK_HIDDEN_SIZE
|
||||
mc.dtype = _MOCK_DTYPE
|
||||
return mc
|
||||
|
||||
|
||||
def test_prompt_embeds_keys_registered():
|
||||
assert "prompt_embeds" in MODALITY_PLACEHOLDERS_MAP
|
||||
assert MODALITY_PLACEHOLDERS_MAP["prompt_embeds"] == "<##PROMPT_EMBEDS##>"
|
||||
assert "prompt_embeds" in MM_PARSER_MAP
|
||||
|
||||
|
||||
def test_ensure_placeholder_token_is_single_token_and_idempotent(tokenizer):
|
||||
"""Ensure the placeholder token is a single token and that multiple calls to
|
||||
"ensure" are idempotent, across all tokenizer families."""
|
||||
tid1 = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
tid2 = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
assert tid1 == tid2
|
||||
|
||||
ids = tokenizer.encode(PROMPT_EMBEDS_PLACEHOLDER_TOKEN, add_special_tokens=False)
|
||||
assert ids == [tid1]
|
||||
|
||||
# Repeating it in a string N times must produce exactly that many tokens.
|
||||
N = 5
|
||||
ids_rep = tokenizer.encode(
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN * N, add_special_tokens=False
|
||||
)
|
||||
assert ids_rep == [tid1] * N
|
||||
|
||||
|
||||
def test_parse_chat_messages_openai_format():
|
||||
NUM_TOKENS = 3
|
||||
t = torch.randn(NUM_TOKENS, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
b64 = _encode_tensor(t)
|
||||
mc = _make_mock_model_config()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Hello "},
|
||||
{"type": "prompt_embeds", "data": b64},
|
||||
{"type": "text", "text": " world"},
|
||||
],
|
||||
}
|
||||
]
|
||||
conv, mm_data, _ = parse_chat_messages(
|
||||
messages,
|
||||
mc,
|
||||
content_format="openai",
|
||||
)
|
||||
# The middle content part is rewritten to a single placeholder-token
|
||||
# sentinel.
|
||||
texts = [p["text"] for p in conv[0]["content"]]
|
||||
assert texts == [
|
||||
"Hello ",
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
|
||||
" world",
|
||||
]
|
||||
assert mm_data is not None and "prompt_embeds" in mm_data
|
||||
assert torch.equal(mm_data["prompt_embeds"][0], t)
|
||||
|
||||
|
||||
# Each layout entry is one content part:
|
||||
# ("text", "A") -> {"type": "text", "text": "A"}
|
||||
# ("embed", N) -> {"type": "prompt_embeds", "data": <base64 of (N, H) tensor>}
|
||||
@pytest.mark.parametrize(
|
||||
"layout",
|
||||
[
|
||||
# Case: Single embed only.
|
||||
[("embed", 2)],
|
||||
# Case: Embed at the start of the message.
|
||||
[("embed", 3), ("text", "B")],
|
||||
# Case: Embed at the end of the message.
|
||||
[("text", "A"), ("embed", 1)],
|
||||
# Case: Embed sandwiched between text spans.
|
||||
[("text", "A"), ("embed", 2), ("text", "B")],
|
||||
# Case: Multiple embeds with text in between.
|
||||
[("text", "A"), ("embed", 2), ("text", "B"), ("embed", 3)],
|
||||
# Case: Adjacent embeds with no separating text.
|
||||
[("embed", 1), ("embed", 2)],
|
||||
# Case: Multiple text spans before a trailing embed.
|
||||
[("text", "A"), ("text", "B"), ("embed", 1)],
|
||||
# Case: Long-ish run mixing both kinds.
|
||||
[
|
||||
("text", "head"),
|
||||
("embed", 4),
|
||||
("text", "mid"),
|
||||
("embed", 1),
|
||||
("embed", 2),
|
||||
("text", "tail"),
|
||||
],
|
||||
],
|
||||
ids=[
|
||||
"single-embed",
|
||||
"embed-then-text",
|
||||
"text-then-embed",
|
||||
"text-embed-text",
|
||||
"text-embed-text-embed",
|
||||
"adjacent-embeds",
|
||||
"text-text-embed",
|
||||
"long-mixed-run",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"interleave_mm_strings",
|
||||
# `None`: text-only path where `multimodal_config` is absent.
|
||||
# `False`: non-interleave multimodal path (the common default).
|
||||
# `True`: sentinel-substitution interleave path.
|
||||
# All three must preserve the request ordering of prompt_embeds
|
||||
# relative to surrounding text because prompt_embeds are spliced at the
|
||||
# token offset during rendering.
|
||||
[None, False, True],
|
||||
ids=["text-only", "interleave-off", "interleave-on"],
|
||||
)
|
||||
def test_parse_chat_messages_string_format_preserves_position(
|
||||
layout, interleave_mm_strings
|
||||
):
|
||||
mc = _make_mock_model_config()
|
||||
if interleave_mm_strings is not None:
|
||||
mm_cfg = mock.MagicMock()
|
||||
mm_cfg.interleave_mm_strings = interleave_mm_strings
|
||||
mc.multimodal_config = mm_cfg
|
||||
|
||||
content: list[dict] = []
|
||||
expected_parts: list[str] = []
|
||||
expected_embeds: list[torch.Tensor] = []
|
||||
for kind, value in layout:
|
||||
if kind == "text":
|
||||
content.append({"type": "text", "text": value})
|
||||
expected_parts.append(value)
|
||||
else: # prompt embeds
|
||||
num_tokens = value
|
||||
t = torch.randn(num_tokens, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
expected_embeds.append(t)
|
||||
content.append({"type": "prompt_embeds", "data": _encode_tensor(t)})
|
||||
# Parser emits ONE sentinel per part.
|
||||
expected_parts.append(PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
|
||||
|
||||
messages = [{"role": "user", "content": content}]
|
||||
conv, mm_data, _ = parse_chat_messages(
|
||||
messages,
|
||||
mc,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
assert conv[0]["content"] == "\n".join(expected_parts)
|
||||
assert mm_data is not None and "prompt_embeds" in mm_data
|
||||
assert len(mm_data["prompt_embeds"]) == len(expected_embeds)
|
||||
for got, want in zip(mm_data["prompt_embeds"], expected_embeds, strict=True):
|
||||
assert torch.equal(got, want)
|
||||
|
||||
|
||||
def test_parse_chat_messages_requires_flag():
|
||||
t = torch.randn(2, 4)
|
||||
b64 = _encode_tensor(t)
|
||||
mc = _make_mock_model_config(enable_prompt_embeds=False)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds", "data": b64}],
|
||||
}
|
||||
]
|
||||
with pytest.raises(VLLMValidationError, match=_ENABLE_PROMPT_EMBEDS_ERROR):
|
||||
parse_chat_messages(
|
||||
messages,
|
||||
mc,
|
||||
content_format="openai",
|
||||
)
|
||||
|
||||
|
||||
def test_parse_chat_messages_rejects_missing_data():
|
||||
# `data` is marked `Required` on `ChatCompletionContentPartPromptEmbedsParam`;
|
||||
# malformed requests without `data` must surface a clear validation error
|
||||
# rather than being silently dropped.
|
||||
mc = _make_mock_model_config()
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "prompt_embeds"}], # no `data`
|
||||
}
|
||||
]
|
||||
with pytest.raises(VLLMValidationError, match=_PROMPT_EMBEDS_MISSING_DATA_ERROR):
|
||||
parse_chat_messages(
|
||||
messages,
|
||||
mc,
|
||||
content_format="openai",
|
||||
)
|
||||
|
||||
|
||||
# Reserved placeholder guard: when `enable_prompt_embeds=True` the tokenizer is
|
||||
# mutated to make `<prompt_embeds>` a single unsplittable token. Any user text
|
||||
# containing that literal sequence would tokenize to the same sentinel ID and
|
||||
# be mistaken for a splice point, so we reject it at parse time.
|
||||
_PLACEHOLDER_ERROR_PATTERN: Final[str] = re.sub(
|
||||
r"\\{[^}]*\\}", ".*", re.escape(_RESERVED_PLACEHOLDER_IN_TEXT_ERROR)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
# Case: Top-level string content (wrapped as a single text part).
|
||||
f"hello {PROMPT_EMBEDS_PLACEHOLDER_TOKEN} world",
|
||||
# Case: List with a typed text part containing the placeholder.
|
||||
[{"type": "text", "text": f"leading {PROMPT_EMBEDS_PLACEHOLDER_TOKEN}"}],
|
||||
# Case: List with a plain-string part (no wrapping dict).
|
||||
[f"raw string {PROMPT_EMBEDS_PLACEHOLDER_TOKEN}"],
|
||||
],
|
||||
ids=["top-level-string", "typed-text-part", "plain-string-part"],
|
||||
)
|
||||
def test_parse_chat_messages_rejects_placeholder_in_user_text(content):
|
||||
mc = _make_mock_model_config() # enable_prompt_embeds=True by default
|
||||
messages = [{"role": "user", "content": content}]
|
||||
with pytest.raises(VLLMValidationError, match=_PLACEHOLDER_ERROR_PATTERN):
|
||||
parse_chat_messages(messages, mc, content_format="openai")
|
||||
|
||||
|
||||
def test_parse_chat_messages_allows_placeholder_in_text_when_feature_disabled():
|
||||
# When `enable_prompt_embeds=False` the tokenizer is never mutated, so the
|
||||
# literal `<prompt_embeds>` is just ordinary text and must pass through.
|
||||
mc = _make_mock_model_config(enable_prompt_embeds=False)
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"benign mention of {PROMPT_EMBEDS_PLACEHOLDER_TOKEN} here",
|
||||
}
|
||||
]
|
||||
conv, mm_data, _ = parse_chat_messages(messages, mc, content_format="openai")
|
||||
assert mm_data is None or "prompt_embeds" not in mm_data
|
||||
# Text reaches the rendered conversation unchanged.
|
||||
texts = [p["text"] for p in conv[0]["content"]]
|
||||
assert PROMPT_EMBEDS_PLACEHOLDER_TOKEN in "".join(texts)
|
||||
|
||||
|
||||
# Token-stream spec: ints are regular token IDs, tuples `(N,)` expand to
|
||||
# a placeholder span of length N (creates corresponding `(N, H)` tensor).
|
||||
# `expected` lists the `(start_idx, length)` pairs that
|
||||
# `_build_prompt_embeds_positions` should return.
|
||||
@pytest.mark.parametrize(
|
||||
"stream, expected",
|
||||
[
|
||||
# Case: Single run in the middle.
|
||||
([10, 20, (3,), 30], [(2, 3)]),
|
||||
# Case: Single run at the start.
|
||||
([(2,), 10, 20], [(0, 2)]),
|
||||
# Case: Single run at the end.
|
||||
([10, 20, (4,)], [(2, 4)]),
|
||||
# Case: Two runs with tokens between.
|
||||
([1, (2,), 2, 3, (3,), 4], [(1, 2), (5, 3)]),
|
||||
# Case: Adjacent runs (no separating tokens).
|
||||
([(1,), (2,)], [(0, 1), (1, 2)]),
|
||||
# Case: Three runs.
|
||||
([5, (2,), 6, (1,), 7, (3,), 8], [(1, 2), (4, 1), (6, 3)]),
|
||||
],
|
||||
ids=[
|
||||
"single-middle",
|
||||
"single-start",
|
||||
"single-end",
|
||||
"two-runs-separated",
|
||||
"two-runs-adjacent",
|
||||
"three-runs",
|
||||
],
|
||||
)
|
||||
def test_build_positions(tokenizer, stream, expected):
|
||||
H = 4
|
||||
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
tensors: list[torch.Tensor] = []
|
||||
token_ids: list[int] = []
|
||||
for item in stream:
|
||||
if isinstance(item, tuple):
|
||||
length = item[0]
|
||||
tensors.append(torch.randn(length, H))
|
||||
token_ids.extend([tid] * length)
|
||||
else:
|
||||
token_ids.append(item)
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, tid)
|
||||
positions = _build_prompt_embeds_positions(token_ids, len(tensors), mm_updates)
|
||||
assert positions == expected
|
||||
|
||||
|
||||
def test_build_positions_length_mismatch(tokenizer):
|
||||
N1, H1 = 2, 4
|
||||
N2, H2 = 3, 4
|
||||
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
# 2 tensors expected but only a single placeholder run in the token
|
||||
# stream (simulating dropping the second one).
|
||||
tensors = [torch.randn(N1, H1), torch.randn(N2, H2)]
|
||||
token_ids = [1, tid, tid, 2, 3]
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, tid)
|
||||
# The error constant is a `str.format` template, escape it and turn
|
||||
# the `{field}` placeholders into `.*` so it matches any substitution.
|
||||
pattern = re.sub(
|
||||
r"\\{[^}]*\\}", ".*", re.escape(_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR)
|
||||
)
|
||||
with pytest.raises(ValueError, match=pattern):
|
||||
_build_prompt_embeds_positions(token_ids, len(tensors), mm_updates)
|
||||
|
||||
|
||||
# ints = regular token IDs (any value)
|
||||
# (N,) = embed span of length N
|
||||
@pytest.mark.parametrize(
|
||||
"stream",
|
||||
[
|
||||
[10, 20, (3,), 30],
|
||||
[(2,), 10, 20],
|
||||
[10, 20, (4,)],
|
||||
[1, (2,), 2, 3, (3,), 4],
|
||||
[(1,), (2,)],
|
||||
[5, (2,), 6, (1,), 7, (3,), 8],
|
||||
],
|
||||
ids=[
|
||||
"single-middle",
|
||||
"single-start",
|
||||
"single-end",
|
||||
"two-spans-separated",
|
||||
"two-spans-adjacent",
|
||||
"three-spans",
|
||||
],
|
||||
)
|
||||
def test_build_mixed_prompt_embeds(stream):
|
||||
H = 8
|
||||
_PLACEHOLDER = 0 # sentinel for embed positions in token_ids
|
||||
|
||||
tensors: list[torch.Tensor] = []
|
||||
token_ids: list[int] = []
|
||||
positions: list[tuple[int, int]] = []
|
||||
cursor = 0
|
||||
for item in stream:
|
||||
if isinstance(item, tuple):
|
||||
length = item[0]
|
||||
tensors.append(torch.randn(length, H))
|
||||
positions.append((cursor, length))
|
||||
token_ids.extend([_PLACEHOLDER] * length)
|
||||
cursor += length
|
||||
else:
|
||||
token_ids.append(item)
|
||||
cursor += 1
|
||||
|
||||
embeds, mask = _build_mixed_prompt_embeds(token_ids, tensors, positions)
|
||||
|
||||
assert embeds.shape == (len(token_ids), H)
|
||||
assert len(mask) == len(token_ids)
|
||||
|
||||
# Mask: False exactly at embed positions, True everywhere else.
|
||||
expected_mask = torch.ones(len(token_ids), dtype=torch.bool)
|
||||
for start, length in positions:
|
||||
expected_mask[start : start + length] = False
|
||||
assert mask == expected_mask.tolist()
|
||||
|
||||
# Embed rows match input tensors at the right positions.
|
||||
for tensor, (start, length) in zip(tensors, positions):
|
||||
assert torch.equal(embeds[start : start + length], tensor)
|
||||
|
||||
# Non-embed positions remain zero-filled.
|
||||
assert torch.all(embeds[expected_mask] == 0)
|
||||
|
||||
|
||||
# End-to-end tests: each runs both sync and async parse paths via the
|
||||
# `parse_fn` fixture.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("role", ["user", "system"])
|
||||
async def test_end_to_end_expand_and_build(tokenizer, parse_fn, role):
|
||||
"""Full renderer pipeline: parse -> chat template -> expand -> locate
|
||||
-> build mixed prompt, across tokenizers, roles, and sync/async."""
|
||||
tokenizer.chat_template = _SIMPLE_CHAT_TEMPLATE
|
||||
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
|
||||
LEN_A, LEN_B = 3, 2
|
||||
t_a = torch.randn(LEN_A, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
t_b = torch.randn(LEN_B, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
NUM_TENSORS = 2
|
||||
|
||||
mc = _make_mock_model_config()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": role,
|
||||
"content": [
|
||||
{"type": "text", "text": "Hello "},
|
||||
{"type": "prompt_embeds", "data": _encode_tensor(t_a)},
|
||||
{"type": "text", "text": " world "},
|
||||
{"type": "prompt_embeds", "data": _encode_tensor(t_b)},
|
||||
{"type": "text", "text": "!"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conv, mm_data, _ = await _maybe_await(
|
||||
parse_fn, messages, mc, content_format="openai"
|
||||
)
|
||||
tensors = list(mm_data["prompt_embeds"])
|
||||
assert len(tensors) == NUM_TENSORS
|
||||
|
||||
# Tokenize: each prompt_embeds part becomes 1 placeholder token.
|
||||
# `return_dict=False` to get a flat `list[int]` on transformers v5
|
||||
# (where the default flipped to True and yields a `BatchEncoding` dict).
|
||||
token_ids = tokenizer.apply_chat_template(conv, tokenize=True, return_dict=False)
|
||||
assert sum(t == tid for t in token_ids) == NUM_TENSORS
|
||||
|
||||
# Expand, locate, and build.
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, tid)
|
||||
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
|
||||
assert len(expanded) == len(token_ids) + LEN_A + LEN_B - NUM_TENSORS
|
||||
|
||||
positions = _build_prompt_embeds_positions(expanded, len(tensors), mm_updates)
|
||||
assert positions[0][1] == LEN_A
|
||||
assert positions[1][1] == LEN_B
|
||||
|
||||
embeds, mask = _build_mixed_prompt_embeds(expanded, tensors, positions)
|
||||
assert embeds.shape == (len(expanded), _MOCK_HIDDEN_SIZE)
|
||||
assert mask.count(False) == LEN_A + LEN_B
|
||||
assert torch.equal(embeds[positions[0][0] : positions[0][0] + LEN_A], t_a)
|
||||
assert torch.equal(embeds[positions[1][0] : positions[1][0] + LEN_B], t_b)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_end_to_end_multi_message_conversation(tokenizer, parse_fn):
|
||||
"""Full pipeline with prompt_embeds spread across system + user messages,
|
||||
verifying ordering and positioning in the final token stream."""
|
||||
tokenizer.chat_template = _SIMPLE_CHAT_TEMPLATE
|
||||
tid = _ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
|
||||
LEN_SYS, LEN_USR = 4, 3
|
||||
t_sys = torch.randn(LEN_SYS, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
t_usr = torch.randn(LEN_USR, _MOCK_HIDDEN_SIZE, dtype=_MOCK_DTYPE)
|
||||
NUM_TENSORS = 2 # t_sys and t_usr.
|
||||
|
||||
mc = _make_mock_model_config()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are helpful."},
|
||||
{"type": "prompt_embeds", "data": _encode_tensor(t_sys)},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "prompt_embeds", "data": _encode_tensor(t_usr)},
|
||||
{"type": "text", "text": "Summarize."},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
conv, mm_data, _ = await _maybe_await(
|
||||
parse_fn, messages, mc, content_format="openai"
|
||||
)
|
||||
tensors = list(mm_data["prompt_embeds"])
|
||||
assert len(tensors) == NUM_TENSORS
|
||||
|
||||
# Tokenize, expand, locate, and build.
|
||||
# `return_dict=False` to get a flat `list[int]` on transformers v5
|
||||
# (where the default flipped to True and yields a `BatchEncoding` dict).
|
||||
token_ids = tokenizer.apply_chat_template(conv, tokenize=True, return_dict=False)
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, tid)
|
||||
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
|
||||
positions = _build_prompt_embeds_positions(expanded, len(tensors), mm_updates)
|
||||
|
||||
assert positions[0][1] == LEN_SYS
|
||||
assert positions[1][1] == LEN_USR
|
||||
# System embed must appear before user embed in the token stream.
|
||||
assert positions[0][0] < positions[1][0]
|
||||
|
||||
embeds, mask = _build_mixed_prompt_embeds(expanded, tensors, positions)
|
||||
assert embeds.shape == (len(expanded), _MOCK_HIDDEN_SIZE)
|
||||
assert mask.count(False) == LEN_SYS + LEN_USR
|
||||
assert torch.equal(embeds[positions[0][0] : positions[0][0] + LEN_SYS], t_sys)
|
||||
assert torch.equal(embeds[positions[1][0] : positions[1][0] + LEN_USR], t_usr)
|
||||
|
||||
|
||||
def test_truncation_keeps_the_mixed_mask_aligned_with_the_prompt():
|
||||
"""`prompt_is_token_ids` must be truncated with `prompt_token_ids`.
|
||||
|
||||
The mask is positional: the engine pairs it element-wise with the prompt
|
||||
tokens to decide which positions read a real token id and which read a row
|
||||
of `prompt_embeds`. Truncating the prompt without truncating the mask
|
||||
leaves the two describing different positions, so embed placeholders stop
|
||||
being recognised as such.
|
||||
"""
|
||||
hidden_size = 8
|
||||
placeholder_token_id = 0
|
||||
num_head, embed_len, num_tail = 10, 5, 5
|
||||
|
||||
token_ids = (
|
||||
list(range(1, num_head + 1))
|
||||
+ [placeholder_token_id] * embed_len
|
||||
+ list(range(1, num_tail + 1))
|
||||
)
|
||||
tensors = [torch.randn(embed_len, hidden_size)]
|
||||
positions = [(num_head, embed_len)]
|
||||
embeds, mask = _build_mixed_prompt_embeds(token_ids, tensors, positions)
|
||||
|
||||
prompt = {
|
||||
"prompt_token_ids": token_ids,
|
||||
"prompt_embeds": embeds,
|
||||
"prompt_is_token_ids": mask,
|
||||
}
|
||||
keep = 8
|
||||
tok_params = TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=keep,
|
||||
truncation_side="left",
|
||||
)
|
||||
|
||||
result = tok_params.apply_post_tokenization(None, prompt)
|
||||
|
||||
# Keeping the last 8 of 20 positions starts inside the embed span, so the
|
||||
# first 3 surviving positions are embed rows and the rest are real tokens.
|
||||
# Comparing contents rather than lengths is what pins the alignment: a
|
||||
# mask reduced from the wrong side has the right length and still
|
||||
# describes the wrong positions.
|
||||
assert result["prompt_is_token_ids"] == [False] * 3 + [True] * num_tail
|
||||
assert result["prompt_token_ids"] == token_ids[-keep:]
|
||||
assert torch.equal(result["prompt_embeds"], embeds[-keep:])
|
||||
@@ -0,0 +1,150 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
|
||||
MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def mistral_model_config():
|
||||
return ModelConfig(
|
||||
MISTRAL_MODEL_ID,
|
||||
runner="generate",
|
||||
limit_mm_per_prompt={
|
||||
"image": 2,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_parse_chat_messages_empty_system(
|
||||
mistral_model_config,
|
||||
):
|
||||
# Test string format
|
||||
conversation, _, _ = parse_chat_messages(
|
||||
[
|
||||
{"role": "system", "content": ""},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "Who are you?"}],
|
||||
},
|
||||
],
|
||||
mistral_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
assert conversation == [
|
||||
{"role": "system", "content": ""},
|
||||
{"role": "user", "content": "Who are you?"},
|
||||
]
|
||||
|
||||
# Test openai format
|
||||
conversation, _, _ = parse_chat_messages(
|
||||
[
|
||||
{"role": "system", "content": ""},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "Who are you?"}],
|
||||
},
|
||||
],
|
||||
mistral_model_config,
|
||||
content_format="openai",
|
||||
)
|
||||
assert conversation == [
|
||||
{"role": "system", "content": [{"type": "text", "text": ""}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "Who are you?"}]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_only_chat_does_not_initialize_media_connector(
|
||||
mistral_model_config,
|
||||
monkeypatch,
|
||||
):
|
||||
load_connector = MagicMock()
|
||||
monkeypatch.setattr(MEDIA_CONNECTOR_REGISTRY, "load", load_connector)
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Who are you?"}
|
||||
]
|
||||
|
||||
parse_chat_messages(
|
||||
messages,
|
||||
mistral_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
await parse_chat_messages_async(
|
||||
messages,
|
||||
mistral_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
load_connector.assert_not_called()
|
||||
|
||||
|
||||
def test_parse_chat_messages_include_thinking_chunk(mistral_model_config):
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are a helpful assistant."},
|
||||
{
|
||||
"type": "thinking",
|
||||
"closed": True,
|
||||
"thinking": "Only return the answer when you are confident.",
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "What is 2+2?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Let me think about it."},
|
||||
{"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "The answer is 4.",
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
conversation_with_thinking, _, _ = parse_chat_messages(
|
||||
messages,
|
||||
mistral_model_config,
|
||||
content_format="openai",
|
||||
)
|
||||
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "You are a helpful assistant."},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Only return the answer when you are confident.",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "What is 2+2?"}],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Let me think about it."},
|
||||
{"type": "text", "text": "2+2 = 4"},
|
||||
{"type": "text", "text": "The answer is 4."},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
assert conversation_with_thinking == expected_conversation
|
||||
@@ -0,0 +1,148 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.renderers.chat_utils import ConversationMessage
|
||||
from vllm.renderers.chat_utils.chat_message import _postprocess_messages
|
||||
|
||||
|
||||
def _assistant_tool_call(arguments, name="write"):
|
||||
return [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_0",
|
||||
"type": "function",
|
||||
"function": {"name": name, "arguments": arguments},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _assistant_tool_calls(calls):
|
||||
return [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": calls,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_tool_call_arguments_dict_passthrough(caplog):
|
||||
messages = _assistant_tool_call({"a": 1})
|
||||
_postprocess_messages(messages)
|
||||
args = messages[0]["tool_calls"][0]["function"]["arguments"]
|
||||
assert args == {"a": 1}
|
||||
assert not caplog.records
|
||||
|
||||
|
||||
def test_tool_call_arguments_valid_object_string(caplog):
|
||||
messages = _assistant_tool_call('{"a": 1}')
|
||||
_postprocess_messages(messages)
|
||||
args = messages[0]["tool_calls"][0]["function"]["arguments"]
|
||||
assert args == {"a": 1}
|
||||
assert not caplog.records
|
||||
|
||||
|
||||
def test_tool_call_arguments_malformed_json_small(caplog):
|
||||
bad = '{"cmd": "mkdir -p /tmp/mmadtest && cat > /tmp/mmadtest'
|
||||
messages = _assistant_tool_call(bad, name="exec")
|
||||
_postprocess_messages(messages)
|
||||
args = messages[0]["tool_calls"][0]["function"]["arguments"]
|
||||
assert args == {}
|
||||
|
||||
assert len(caplog.records) == 1
|
||||
assert "coercing to an empty object" in caplog.records[0].message
|
||||
|
||||
|
||||
def test_tool_call_arguments_malformed_json_large(caplog):
|
||||
bad = '{"filepath": "src/main.py", "contents": "' + ("x" * 18000)
|
||||
messages = _assistant_tool_call(bad, name="write")
|
||||
_postprocess_messages(messages)
|
||||
args = messages[0]["tool_calls"][0]["function"]["arguments"]
|
||||
assert args == {}
|
||||
|
||||
assert len(caplog.records) == 1
|
||||
assert "coercing to an empty object" in caplog.records[0].message
|
||||
assert f"{len(bad)} chars" in caplog.records[0].message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("non_obj", ["[]", "42", "true", '"hello"', "null"])
|
||||
def test_tool_call_arguments_valid_json_non_object(caplog, non_obj):
|
||||
messages = _assistant_tool_call(non_obj, name="bad_tool")
|
||||
_postprocess_messages(messages)
|
||||
args = messages[0]["tool_calls"][0]["function"]["arguments"]
|
||||
assert args == {}
|
||||
|
||||
if non_obj == "null":
|
||||
# null parses to None, so no warning is emitted according to requirements
|
||||
assert not caplog.records
|
||||
else:
|
||||
assert len(caplog.records) == 1
|
||||
assert "not a JSON object" in caplog.records[0].message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing", [None, ""])
|
||||
def test_tool_call_arguments_missing_or_empty(caplog, missing):
|
||||
messages = _assistant_tool_call(missing)
|
||||
if missing is None:
|
||||
del messages[0]["tool_calls"][0]["function"]["arguments"]
|
||||
_postprocess_messages(messages)
|
||||
args = messages[0]["tool_calls"][0]["function"]["arguments"]
|
||||
assert args == {}
|
||||
assert not caplog.records
|
||||
|
||||
|
||||
def test_tool_call_arguments_multiple_independent(caplog):
|
||||
calls = [
|
||||
{
|
||||
"id": "call_0",
|
||||
"type": "function",
|
||||
"function": {"name": "good", "arguments": '{"a": 1}'},
|
||||
},
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "bad", "arguments": '{"cmd": "'},
|
||||
},
|
||||
]
|
||||
messages = _assistant_tool_calls(calls)
|
||||
_postprocess_messages(messages)
|
||||
|
||||
tool_calls = messages[0]["tool_calls"]
|
||||
assert tool_calls[0]["function"]["arguments"] == {"a": 1}
|
||||
assert tool_calls[1]["function"]["arguments"] == {}
|
||||
|
||||
assert len(caplog.records) == 1
|
||||
assert "coercing to an empty object" in caplog.records[0].message
|
||||
|
||||
|
||||
def test_postprocess_messages_null_arguments_string():
|
||||
"""arguments="null" must not reach the chat template as Python None.
|
||||
|
||||
json.loads("null") returns None, which causes Jinja2 templates that call
|
||||
tc.arguments.items() to raise 'None' has no attribute 'items'.
|
||||
The function should coerce it to {} instead.
|
||||
"""
|
||||
messages: list[ConversationMessage] = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_current_time", "arguments": "null"},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
_postprocess_messages(messages)
|
||||
tool_calls = messages[0]["tool_calls"]
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0]["function"]["arguments"] == {}
|
||||
@@ -0,0 +1,497 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers.chat_utils import parse_chat_messages, parse_chat_messages_async
|
||||
from vllm.utils.serial_utils import tensor2base64
|
||||
|
||||
from .utils import (
|
||||
assert_mm_data_inputs,
|
||||
assert_mm_uuids,
|
||||
)
|
||||
|
||||
QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def qwen25omni_model_config_image_embeds():
|
||||
return ModelConfig(
|
||||
QWEN25OMNI_MODEL_ID,
|
||||
runner="generate",
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
enable_mm_embeds=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def qwen25omni_model_config_video_embeds():
|
||||
return ModelConfig(
|
||||
QWEN25OMNI_MODEL_ID,
|
||||
runner="generate",
|
||||
limit_mm_per_prompt={"video": 2},
|
||||
enable_mm_embeds=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def qwen25omni_model_config_mm_interleaved():
|
||||
return ModelConfig(
|
||||
QWEN25OMNI_MODEL_ID,
|
||||
runner="generate",
|
||||
interleave_mm_strings=True,
|
||||
limit_mm_per_prompt={
|
||||
"image": 2,
|
||||
"audio": 1,
|
||||
"video": 1,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_parse_chat_messages_empty_video_embeds_with_uuid(
|
||||
qwen25omni_model_config_video_embeds,
|
||||
):
|
||||
"""Test video_embeds with UUID (no actual embeds data)."""
|
||||
uuid = "test-video-uuid-123"
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this video"},
|
||||
{"type": "video_embeds", "video_embeds": None, "uuid": uuid},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_video_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Should have video in mm_data as None (UUID provided)
|
||||
assert mm_data is not None
|
||||
assert "video" in mm_data
|
||||
assert isinstance(mm_data["video"], list)
|
||||
assert len(mm_data["video"]) == 1
|
||||
assert mm_data["video"][0] is None
|
||||
|
||||
# UUID should be recorded
|
||||
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[uuid])
|
||||
|
||||
|
||||
def test_parse_chat_messages_video_embeds_with_string(
|
||||
qwen25omni_model_config_video_embeds,
|
||||
):
|
||||
"""Test video_embeds with base64 string embedding data."""
|
||||
hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()
|
||||
video_embedding = torch.randn(1, 128, hidden_size)
|
||||
|
||||
base64_video_embedding = tensor2base64(video_embedding)
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this video"},
|
||||
{
|
||||
"type": "video_embeds",
|
||||
"video_embeds": base64_video_embedding,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_video_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Should have video embedding in mm_data (single tensor, not a list)
|
||||
assert mm_data is not None
|
||||
assert "video" in mm_data
|
||||
assert isinstance(mm_data["video"], torch.Tensor)
|
||||
assert mm_data["video"].shape == video_embedding.shape
|
||||
# No UUID provided
|
||||
assert mm_uuids is not None
|
||||
assert "video" in mm_uuids
|
||||
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_chat_messages_video_embeds_async(
|
||||
qwen25omni_model_config_video_embeds,
|
||||
):
|
||||
"""Test video_embeds with async futures."""
|
||||
hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()
|
||||
video_embedding = torch.randn(1, 128, hidden_size)
|
||||
|
||||
base64_video_embedding = tensor2base64(video_embedding)
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this video"},
|
||||
{
|
||||
"type": "video_embeds",
|
||||
"video_embeds": base64_video_embedding,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_video_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Should have video embedding in mm_data (single tensor, not a list)
|
||||
assert mm_data is not None
|
||||
assert "video" in mm_data
|
||||
assert isinstance(mm_data["video"], torch.Tensor)
|
||||
assert mm_data["video"].shape == video_embedding.shape
|
||||
# No UUID provided
|
||||
assert mm_uuids is not None
|
||||
assert "video" in mm_uuids
|
||||
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
|
||||
|
||||
|
||||
def test_parse_chat_messages_video_and_video_embeds_mixed(
|
||||
qwen25omni_model_config_video_embeds,
|
||||
video_url,
|
||||
):
|
||||
"""Test that mixing raw video and video_embeds is rejected."""
|
||||
with pytest.raises(VLLMValidationError, match="Mixing raw video"):
|
||||
parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "video_url", "video_url": {"url": video_url}},
|
||||
{"type": "video_embeds", "video_embeds": None, "uuid": "x"},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_video_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("json_metadata", [False, True])
|
||||
def test_parse_chat_messages_multiple_dict_image_embeds(
|
||||
qwen25omni_model_config_image_embeds,
|
||||
json_metadata,
|
||||
):
|
||||
"""Test that multiple dictionaries for image_embeds is handled without errors."""
|
||||
# Create two sample image embedding tensors
|
||||
batch_size = 2
|
||||
hidden_size = qwen25omni_model_config_image_embeds.get_inputs_embeds_size()
|
||||
image_embeds = torch.randn(batch_size * 220, hidden_size)
|
||||
image_grid_thw = torch.tensor([[1, 22, 40] for _ in range(batch_size)])
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_embeds",
|
||||
"image_embeds": {
|
||||
"image_embeds": tensor2base64(embeds),
|
||||
"image_grid_thw": grid_thw.tolist()
|
||||
if json_metadata
|
||||
else tensor2base64(grid_thw),
|
||||
},
|
||||
}
|
||||
for embeds, grid_thw in zip(
|
||||
image_embeds.chunk(batch_size), image_grid_thw
|
||||
)
|
||||
]
|
||||
+ [
|
||||
{"type": "text", "text": "Describe these two images."},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_image_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Verify conversation structure
|
||||
assert conversation == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "<|vision_start|><|IMAGE|><|vision_end|>\n"
|
||||
"<|vision_start|><|IMAGE|><|vision_end|>\nDescribe these two images.",
|
||||
}
|
||||
]
|
||||
|
||||
# Verify mm_data contains a dictionary of multi-embeddings
|
||||
assert mm_data is not None
|
||||
assert "image" in mm_data
|
||||
assert isinstance(mm_data["image"], dict)
|
||||
assert len(mm_data["image"]) == batch_size
|
||||
|
||||
# Verify each embedding has the correct shape
|
||||
assert isinstance(mm_data["image"]["image_embeds"], torch.Tensor)
|
||||
assert mm_data["image"]["image_embeds"].shape == image_embeds.shape
|
||||
assert isinstance(mm_data["image"]["image_grid_thw"], torch.Tensor)
|
||||
assert mm_data["image"]["image_grid_thw"].shape == image_grid_thw.shape
|
||||
|
||||
# Verify UUIDs (None since we didn't provide any)
|
||||
assert_mm_uuids(mm_uuids, batch_size, expected_uuids=[None, None])
|
||||
|
||||
|
||||
def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
|
||||
qwen25omni_model_config_mm_interleaved,
|
||||
image_url,
|
||||
video_url,
|
||||
audio_url,
|
||||
):
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's on this image?"},
|
||||
{"type": "image_url", "image_url": {"url": image_url}},
|
||||
{"type": "text", "text": "Now listen to this audio"},
|
||||
{"type": "audio_url", "audio_url": {"url": audio_url}},
|
||||
],
|
||||
},
|
||||
{"role": "assistant", "content": "Some stuff."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's on this image?"},
|
||||
{"type": "image_url", "image_url": {"url": image_url}},
|
||||
{"type": "text", "text": "And what's in the video?"},
|
||||
{"type": "video_url", "video_url": {"url": video_url}},
|
||||
],
|
||||
},
|
||||
],
|
||||
qwen25omni_model_config_mm_interleaved,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
assert conversation == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
|
||||
"\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
|
||||
},
|
||||
{"role": "assistant", "content": "Some stuff."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
|
||||
"\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
|
||||
},
|
||||
]
|
||||
|
||||
assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
|
||||
assert_mm_uuids(mm_uuids, 2, modality="image", expected_uuids=[None, None])
|
||||
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
|
||||
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
|
||||
|
||||
|
||||
def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interleave(
|
||||
qwen25omni_model_config_mm_interleaved,
|
||||
image_url,
|
||||
video_url,
|
||||
audio_url,
|
||||
):
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's on this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
"uuid": "image_123",
|
||||
},
|
||||
{"type": "text", "text": "Now listen to this audio"},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": audio_url},
|
||||
"uuid": "audio_123",
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "assistant", "content": "Some stuff."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's on this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
"uuid": "image_123",
|
||||
},
|
||||
{"type": "text", "text": "And what's in the video?"},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
"uuid": "video_123",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
qwen25omni_model_config_mm_interleaved,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
assert conversation == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
|
||||
"\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
|
||||
},
|
||||
{"role": "assistant", "content": "Some stuff."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
|
||||
"\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
|
||||
},
|
||||
]
|
||||
|
||||
assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
|
||||
assert_mm_uuids(
|
||||
mm_uuids, 2, modality="image", expected_uuids=["image_123", "image_123"]
|
||||
)
|
||||
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
|
||||
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=["audio_123"])
|
||||
|
||||
|
||||
def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_messages_interleave( # noqa: E501
|
||||
qwen25omni_model_config_mm_interleaved,
|
||||
image_url,
|
||||
video_url,
|
||||
audio_url,
|
||||
):
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's on this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": None,
|
||||
"uuid": "image_123",
|
||||
},
|
||||
{"type": "text", "text": "Now listen to this audio"},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": None,
|
||||
"uuid": "audio_123",
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "assistant", "content": "Some stuff."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's on this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": None,
|
||||
"uuid": "image_123",
|
||||
},
|
||||
{"type": "text", "text": "And what's in the video?"},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": None,
|
||||
"uuid": "video_123",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
qwen25omni_model_config_mm_interleaved,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
assert conversation == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
|
||||
"\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
|
||||
},
|
||||
{"role": "assistant", "content": "Some stuff."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
|
||||
"\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
|
||||
},
|
||||
]
|
||||
|
||||
assert_mm_data_inputs(
|
||||
mm_data,
|
||||
{"image": 2, "video": 1, "audio": 1},
|
||||
skipped_media_indices={"image": [0, 1], "video": [0], "audio": [0]},
|
||||
)
|
||||
assert_mm_uuids(
|
||||
mm_uuids, 2, modality="image", expected_uuids=["image_123", "image_123"]
|
||||
)
|
||||
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
|
||||
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=["audio_123"])
|
||||
|
||||
|
||||
def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_messages_interleave( # noqa: E501
|
||||
qwen25omni_model_config_mm_interleaved,
|
||||
image_url,
|
||||
video_url,
|
||||
audio_url,
|
||||
):
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's on this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
"uuid": "image_123",
|
||||
},
|
||||
{"type": "text", "text": "Now listen to this audio"},
|
||||
{"type": "audio_url", "audio_url": {"url": audio_url}},
|
||||
],
|
||||
},
|
||||
{"role": "assistant", "content": "Some stuff."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's on this image?"},
|
||||
{"type": "image_url", "image_url": {"url": image_url}},
|
||||
{"type": "text", "text": "And what's in the video?"},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
"uuid": "video_123",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
qwen25omni_model_config_mm_interleaved,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
assert conversation == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
|
||||
"\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
|
||||
},
|
||||
{"role": "assistant", "content": "Some stuff."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
|
||||
"\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
|
||||
},
|
||||
]
|
||||
|
||||
assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
|
||||
assert_mm_uuids(mm_uuids, 2, modality="image", expected_uuids=["image_123", None])
|
||||
assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
|
||||
assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
|
||||
@@ -0,0 +1,546 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
|
||||
from .utils import assert_mm_uuids
|
||||
|
||||
KIMI_K2_5_MODEL_ID = "moonshotai/Kimi-K2.5"
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def kimi_k2_5_model_config():
|
||||
return ModelConfig(
|
||||
KIMI_K2_5_MODEL_ID,
|
||||
runner="generate",
|
||||
trust_remote_code=True,
|
||||
limit_mm_per_prompt={
|
||||
"image": 2,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _assert_mm_data_is_vision_chunk_input(
|
||||
mm_data: MultiModalDataDict | None,
|
||||
vision_chunk_count: int,
|
||||
) -> None:
|
||||
assert mm_data is not None
|
||||
assert set(mm_data.keys()) == {"vision_chunk"}
|
||||
|
||||
vision_chunk_data = mm_data.get("vision_chunk")
|
||||
assert vision_chunk_data is not None
|
||||
|
||||
assert (
|
||||
isinstance(vision_chunk_data, list)
|
||||
and len(vision_chunk_data) == vision_chunk_count
|
||||
)
|
||||
|
||||
|
||||
def test_parse_chat_messages_image_vision_chunk(
|
||||
kimi_k2_5_model_config,
|
||||
image_url,
|
||||
):
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Analyze this image."},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
kimi_k2_5_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
placeholder = "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"{placeholder}\nAnalyze this image.",
|
||||
}
|
||||
]
|
||||
|
||||
assert conversation == expected_conversation
|
||||
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
|
||||
assert_mm_uuids(mm_uuids, 1, expected_uuids=[None], modality="vision_chunk")
|
||||
|
||||
|
||||
def test_parse_chat_messages_video_vision_chunk(
|
||||
kimi_k2_5_model_config,
|
||||
video_url,
|
||||
):
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Analyze this video."},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
kimi_k2_5_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
placeholder = "<|kimi_k25_video_placeholder|>"
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"{placeholder}\nAnalyze this video.",
|
||||
}
|
||||
]
|
||||
|
||||
assert conversation == expected_conversation
|
||||
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
|
||||
assert_mm_uuids(mm_uuids, 1, expected_uuids=[None], modality="vision_chunk")
|
||||
|
||||
|
||||
def test_parse_chat_messages_image_vision_chunk_with_uuid(
|
||||
kimi_k2_5_model_config,
|
||||
image_url,
|
||||
):
|
||||
image_uuid = "image_123"
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Analyze this image."},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
"uuid": image_uuid,
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
kimi_k2_5_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
placeholder = "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"{placeholder}\nAnalyze this image.",
|
||||
}
|
||||
]
|
||||
|
||||
assert conversation == expected_conversation
|
||||
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
|
||||
assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid], modality="vision_chunk")
|
||||
|
||||
|
||||
def test_parse_chat_messages_video_vision_chunk_with_uuid(
|
||||
kimi_k2_5_model_config,
|
||||
video_url,
|
||||
):
|
||||
video_uuid = "video_456"
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Analyze this video."},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
"uuid": video_uuid,
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
kimi_k2_5_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
placeholder = "<|kimi_k25_video_placeholder|>"
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"{placeholder}\nAnalyze this video.",
|
||||
}
|
||||
]
|
||||
|
||||
assert conversation == expected_conversation
|
||||
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
|
||||
assert_mm_uuids(mm_uuids, 1, expected_uuids=[video_uuid], modality="vision_chunk")
|
||||
|
||||
|
||||
def test_parse_chat_messages_mixed_vision_chunk(
|
||||
kimi_k2_5_model_config,
|
||||
image_url,
|
||||
video_url,
|
||||
):
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Analyze this image and video."},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
kimi_k2_5_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
image_placeholder = (
|
||||
"<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
|
||||
)
|
||||
video_placeholder = "<|kimi_k25_video_placeholder|>"
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"{image_placeholder}\n{video_placeholder}\n"
|
||||
"Analyze this image and video."
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
assert conversation == expected_conversation
|
||||
_assert_mm_data_is_vision_chunk_input(mm_data, 2)
|
||||
assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None], modality="vision_chunk")
|
||||
|
||||
|
||||
def test_parse_chat_messages_mixed_vision_chunk_with_uuid(
|
||||
kimi_k2_5_model_config,
|
||||
image_url,
|
||||
video_url,
|
||||
):
|
||||
image_uuid = "image_123"
|
||||
video_uuid = "video_456"
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Analyze this image and video."},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
"uuid": image_uuid,
|
||||
},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
"uuid": video_uuid,
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
kimi_k2_5_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
image_placeholder = (
|
||||
"<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
|
||||
)
|
||||
video_placeholder = "<|kimi_k25_video_placeholder|>"
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"{image_placeholder}\n{video_placeholder}\n"
|
||||
"Analyze this image and video."
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
assert conversation == expected_conversation
|
||||
_assert_mm_data_is_vision_chunk_input(mm_data, 2)
|
||||
assert_mm_uuids(
|
||||
mm_uuids, 2, expected_uuids=[image_uuid, video_uuid], modality="vision_chunk"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_chat_messages_mixed_vision_chunk_async(
|
||||
kimi_k2_5_model_config,
|
||||
image_url,
|
||||
video_url,
|
||||
):
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Analyze this image and video."},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
kimi_k2_5_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
image_placeholder = (
|
||||
"<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
|
||||
)
|
||||
video_placeholder = "<|kimi_k25_video_placeholder|>"
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"{image_placeholder}\n{video_placeholder}\n"
|
||||
"Analyze this image and video."
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
assert conversation == expected_conversation
|
||||
_assert_mm_data_is_vision_chunk_input(mm_data, 2)
|
||||
assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None], modality="vision_chunk")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_chat_messages_mixed_vision_chunk_with_uuid_async(
|
||||
kimi_k2_5_model_config,
|
||||
image_url,
|
||||
video_url,
|
||||
):
|
||||
image_uuid = "image_123"
|
||||
video_uuid = "video_456"
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Analyze this image and video."},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
"uuid": image_uuid,
|
||||
},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
"uuid": video_uuid,
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
kimi_k2_5_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
image_placeholder = (
|
||||
"<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
|
||||
)
|
||||
video_placeholder = "<|kimi_k25_video_placeholder|>"
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"{image_placeholder}\n{video_placeholder}\n"
|
||||
"Analyze this image and video."
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
assert conversation == expected_conversation
|
||||
_assert_mm_data_is_vision_chunk_input(mm_data, 2)
|
||||
assert_mm_uuids(
|
||||
mm_uuids, 2, expected_uuids=[image_uuid, video_uuid], modality="vision_chunk"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_chat_messages_image_vision_chunk_async(
|
||||
kimi_k2_5_model_config,
|
||||
image_url,
|
||||
):
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Analyze this image."},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
kimi_k2_5_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
placeholder = "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"{placeholder}\nAnalyze this image.",
|
||||
}
|
||||
]
|
||||
|
||||
assert conversation == expected_conversation
|
||||
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
|
||||
assert_mm_uuids(mm_uuids, 1, expected_uuids=[None], modality="vision_chunk")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_chat_messages_video_vision_chunk_async(
|
||||
kimi_k2_5_model_config,
|
||||
video_url,
|
||||
):
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Analyze this video."},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
kimi_k2_5_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
placeholder = "<|kimi_k25_video_placeholder|>"
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"{placeholder}\nAnalyze this video.",
|
||||
}
|
||||
]
|
||||
|
||||
assert conversation == expected_conversation
|
||||
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
|
||||
assert_mm_uuids(mm_uuids, 1, expected_uuids=[None], modality="vision_chunk")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_chat_messages_image_vision_chunk_with_uuid_async(
|
||||
kimi_k2_5_model_config,
|
||||
image_url,
|
||||
):
|
||||
image_uuid = "image_123"
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Analyze this image."},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
"uuid": image_uuid,
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
kimi_k2_5_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
placeholder = "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"{placeholder}\nAnalyze this image.",
|
||||
}
|
||||
]
|
||||
|
||||
assert conversation == expected_conversation
|
||||
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
|
||||
assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid], modality="vision_chunk")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_chat_messages_video_vision_chunk_with_uuid_async(
|
||||
kimi_k2_5_model_config,
|
||||
video_url,
|
||||
):
|
||||
video_uuid = "video_456"
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Analyze this video."},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": video_url},
|
||||
"uuid": video_uuid,
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
kimi_k2_5_model_config,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
placeholder = "<|kimi_k25_video_placeholder|>"
|
||||
expected_conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"{placeholder}\nAnalyze this video.",
|
||||
}
|
||||
]
|
||||
|
||||
assert conversation == expected_conversation
|
||||
_assert_mm_data_is_vision_chunk_input(mm_data, 1)
|
||||
assert_mm_uuids(mm_uuids, 1, expected_uuids=[video_uuid], modality="vision_chunk")
|
||||
@@ -0,0 +1,66 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping
|
||||
from typing import Literal
|
||||
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
|
||||
|
||||
ModalityType = Literal["image", "video", "audio"]
|
||||
MultiModalDataCounts = Mapping[ModalityType, int]
|
||||
|
||||
|
||||
def assert_mm_uuids(
|
||||
mm_uuids: MultiModalUUIDDict | None,
|
||||
media_count: int,
|
||||
expected_uuids: list[str | None],
|
||||
modality: str = "image",
|
||||
) -> None:
|
||||
if len(expected_uuids) > 0:
|
||||
assert mm_uuids is not None
|
||||
assert modality in mm_uuids
|
||||
|
||||
image_uuids = mm_uuids.get(modality)
|
||||
assert image_uuids is not None
|
||||
|
||||
assert isinstance(image_uuids, list) and len(image_uuids) == media_count
|
||||
|
||||
assert image_uuids == expected_uuids
|
||||
else:
|
||||
assert mm_uuids is None
|
||||
|
||||
|
||||
def assert_mm_data_inputs(
|
||||
mm_data: MultiModalDataDict | None,
|
||||
data_count: MultiModalDataCounts,
|
||||
skipped_media_indices: dict[str, list] | None = None, # modality -> list[int]
|
||||
) -> None:
|
||||
assert mm_data is not None
|
||||
assert set(data_count.keys()) == (set(mm_data.keys()))
|
||||
|
||||
for modality, n in data_count.items():
|
||||
modality_data = mm_data.get(modality)
|
||||
assert modality_data is not None
|
||||
assert isinstance(modality_data, list) and len(modality_data) == n
|
||||
|
||||
if skipped_media_indices is not None:
|
||||
skipped_media_indices_for_modality = skipped_media_indices.get(modality)
|
||||
assert skipped_media_indices_for_modality is not None
|
||||
for i in skipped_media_indices_for_modality:
|
||||
assert modality_data[i] is None
|
||||
|
||||
|
||||
def assert_mm_data_is_image_input(
|
||||
mm_data: MultiModalDataDict | None,
|
||||
image_count: int,
|
||||
skipped_image_indices: list | None = None,
|
||||
) -> None:
|
||||
assert mm_data is not None
|
||||
assert set(mm_data.keys()) == {"image"}
|
||||
|
||||
image_data = mm_data.get("image")
|
||||
assert image_data is not None
|
||||
|
||||
assert isinstance(image_data, list) and len(image_data) == image_count
|
||||
if skipped_image_indices is not None:
|
||||
for i in skipped_image_indices:
|
||||
assert image_data[i] is None
|
||||
Reference in New Issue
Block a user