From ec32f669bb557e85d3f05a814d4e4a68f7e3b634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kwieci=C5=84ski?= <153385549+Jankwi@users.noreply.github.com> Date: Tue, 1 Sep 2026 07:55:27 +0200 Subject: [PATCH] [Feature][MM_UUIDs] Allow empty video URLs when using multi-modal UUIDs (#54220) Signed-off-by: Jan Kwiecinski --- docs/features/multimodal_inputs.md | 7 ++ .../openai/chat_completion/test_video.py | 112 ++++++++++++++++++ tests/multimodal/test_parse.py | 21 +++- vllm/multimodal/parse.py | 7 ++ 4 files changed, 146 insertions(+), 1 deletion(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index dd7ffec50c..a36e40c3b9 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -1366,4 +1366,11 @@ Just like with offline inference, you can skip sending media if you expect cache "uuid": image_uuid, }, + # video_url: + { + "type": "video_url", + "video_url": {}, + "uuid": video_uuid, + }, + ``` diff --git a/tests/entrypoints/multimodal/openai/chat_completion/test_video.py b/tests/entrypoints/multimodal/openai/chat_completion/test_video.py index a8c6e472b9..682eb97386 100644 --- a/tests/entrypoints/multimodal/openai/chat_completion/test_video.py +++ b/tests/entrypoints/multimodal/openai/chat_completion/test_video.py @@ -66,6 +66,77 @@ def url_encoded_video() -> dict[str, str]: } +def assert_non_empty_content(chat_completion, *, context: str = "") -> str: + """Assert the first choice has non-empty string content; return it.""" + prefix = f"[{context}] " if context else "" + choice = chat_completion.choices[0] + content = choice.message.content + + assert content is not None, ( + f"{prefix}Expected non-None content but got None. " + f"finish_reason={choice.finish_reason!r}, " + f"full message={choice.message!r}, " + f"usage={chat_completion.usage!r}" + ) + assert isinstance(content, str), ( + f"{prefix}Expected str content, got {type(content).__name__}: {content!r}" + ) + assert len(content) > 0, ( + f"{prefix}Expected non-empty content but got empty string. " + f"finish_reason={choice.finish_reason!r}, " + f"full message={choice.message!r}, " + f"usage={chat_completion.usage!r}" + ) + return content + + +def describe_video_messages( + video_url: str | None, *, extra_video_fields: dict | None = None +) -> list[dict]: + """Build the system + user messages used by the completions-with-video + family of tests. *extra_video_fields* is merged into the top-level + video content block (for uuid / bad-key tests). + + Pass ``None`` for *video_url* to omit media (uuid-backed cache hits). + """ + video_block: dict = { + "type": "video_url", + "video_url": {} if video_url is None else {"url": video_url}, + } + if extra_video_fields: + video_block.update(extra_video_fields) + + return [ + {"role": "system", "content": "You are a helpful assistant."}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + video_block, + ], + }, + ] + + +async def complete_and_check( + client: openai.AsyncOpenAI, + model_name: str, + messages: list[dict], + *, + context: str, + max_completion_tokens: int = 50, + temperature: float = 0.0, +) -> str: + """Run a chat completion and assert the output is non-empty.""" + chat_completion = await client.chat.completions.create( + model=model_name, + messages=messages, + max_completion_tokens=max_completion_tokens, + temperature=temperature, + ) + return assert_non_empty_content(chat_completion, context=context) + + def dummy_messages_from_video_url( video_urls: str | list[str], content_text: str = "What's in this video?", @@ -402,3 +473,44 @@ async def test_multi_video_input( ) message = chat_completion.choices[0].message assert message.content is not None and len(message.content) >= 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", [MODEL_NAME]) +async def test_omitted_video_url_requires_cached_uuid( + client: openai.AsyncOpenAI, + model_name: str, +): + video_uuid = "test-video-uuid" + video_url = TEST_VIDEO_URLS[0] + + omitted_video_messages = describe_video_messages( + None, + extra_video_fields={"uuid": video_uuid}, + ) + + with pytest.raises( + openai.BadRequestError, + match="Cache miss for video at index 0 but data is not provided", + ): + await client.chat.completions.create( + model=model_name, + messages=omitted_video_messages, + ) + + await complete_and_check( + client, + model_name, + describe_video_messages( + video_url, + extra_video_fields={"uuid": video_uuid}, + ), + context="populate UUID cache", + ) + + await complete_and_check( + client, + model_name, + omitted_video_messages, + context="reuse cached video without URL", + ) diff --git a/tests/multimodal/test_parse.py b/tests/multimodal/test_parse.py index 6504cc6bcf..685e615f78 100644 --- a/tests/multimodal/test_parse.py +++ b/tests/multimodal/test_parse.py @@ -5,7 +5,11 @@ import pytest import torch from PIL import Image -from vllm.multimodal.parse import ImageProcessorItems, VideoProcessorItems +from vllm.multimodal.parse import ( + ImageProcessorItems, + MultiModalDataParser, + VideoProcessorItems, +) H, W = 480, 640 @@ -49,3 +53,18 @@ def test_frame_size_hwc_chw(frame): items = VideoProcessorItems([[frame]]) assert items.get_frame_size(0) == (W, H) + + +@pytest.mark.parametrize( + "modality,processor_cls", + [ + ("image", ImageProcessorItems), + ("video", VideoProcessorItems), + ], +) +def test_parse_mm_data_accepts_none_cached_item(modality, processor_cls): + mm_items = MultiModalDataParser().parse_mm_data({modality: [None]}) + items = mm_items[modality] + assert isinstance(items, processor_cls) + assert len(items) == 1 + assert items.get(0) is None diff --git a/vllm/multimodal/parse.py b/vllm/multimodal/parse.py index 6306d6a136..7a4ffc3d6d 100644 --- a/vllm/multimodal/parse.py +++ b/vllm/multimodal/parse.py @@ -756,9 +756,16 @@ class MultiModalDataParser: np.ndarray | MediaWithBytes[np.ndarray] | tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any]] + | None ]() metadata_lst: list[dict[str, Any] | None] = [] for data_item in data_items: + # Allow None video items, valid requests can contain empty URLs + # if they use multi-modal uuids. + if data_item is None: + new_videos.append(None) + metadata_lst.append(None) + continue video, metadata = self._get_video_with_metadata(data_item) if self.video_needs_metadata: if metadata is None: