[Feature][MM_UUIDs] Allow empty video URLs when using multi-modal UUIDs (#54220)
Signed-off-by: Jan Kwiecinski <jkwiecinski@nvidia.com>
This commit is contained in:
@@ -1366,4 +1366,11 @@ Just like with offline inference, you can skip sending media if you expect cache
|
|||||||
"uuid": image_uuid,
|
"uuid": image_uuid,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
# video_url:
|
||||||
|
{
|
||||||
|
"type": "video_url",
|
||||||
|
"video_url": {},
|
||||||
|
"uuid": video_uuid,
|
||||||
|
},
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -66,6 +66,77 @@ def url_encoded_video() -> dict[str, str]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def assert_non_empty_content(chat_completion, *, context: str = "") -> str:
|
||||||
|
"""Assert the first choice has non-empty string content; return it."""
|
||||||
|
prefix = f"[{context}] " if context else ""
|
||||||
|
choice = chat_completion.choices[0]
|
||||||
|
content = choice.message.content
|
||||||
|
|
||||||
|
assert content is not None, (
|
||||||
|
f"{prefix}Expected non-None content but got None. "
|
||||||
|
f"finish_reason={choice.finish_reason!r}, "
|
||||||
|
f"full message={choice.message!r}, "
|
||||||
|
f"usage={chat_completion.usage!r}"
|
||||||
|
)
|
||||||
|
assert isinstance(content, str), (
|
||||||
|
f"{prefix}Expected str content, got {type(content).__name__}: {content!r}"
|
||||||
|
)
|
||||||
|
assert len(content) > 0, (
|
||||||
|
f"{prefix}Expected non-empty content but got empty string. "
|
||||||
|
f"finish_reason={choice.finish_reason!r}, "
|
||||||
|
f"full message={choice.message!r}, "
|
||||||
|
f"usage={chat_completion.usage!r}"
|
||||||
|
)
|
||||||
|
return content
|
||||||
|
|
||||||
|
|
||||||
|
def describe_video_messages(
|
||||||
|
video_url: str | None, *, extra_video_fields: dict | None = None
|
||||||
|
) -> list[dict]:
|
||||||
|
"""Build the system + user messages used by the completions-with-video
|
||||||
|
family of tests. *extra_video_fields* is merged into the top-level
|
||||||
|
video content block (for uuid / bad-key tests).
|
||||||
|
|
||||||
|
Pass ``None`` for *video_url* to omit media (uuid-backed cache hits).
|
||||||
|
"""
|
||||||
|
video_block: dict = {
|
||||||
|
"type": "video_url",
|
||||||
|
"video_url": {} if video_url is None else {"url": video_url},
|
||||||
|
}
|
||||||
|
if extra_video_fields:
|
||||||
|
video_block.update(extra_video_fields)
|
||||||
|
|
||||||
|
return [
|
||||||
|
{"role": "system", "content": "You are a helpful assistant."},
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
{"type": "text", "text": "Describe this video."},
|
||||||
|
video_block,
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
async def complete_and_check(
|
||||||
|
client: openai.AsyncOpenAI,
|
||||||
|
model_name: str,
|
||||||
|
messages: list[dict],
|
||||||
|
*,
|
||||||
|
context: str,
|
||||||
|
max_completion_tokens: int = 50,
|
||||||
|
temperature: float = 0.0,
|
||||||
|
) -> str:
|
||||||
|
"""Run a chat completion and assert the output is non-empty."""
|
||||||
|
chat_completion = await client.chat.completions.create(
|
||||||
|
model=model_name,
|
||||||
|
messages=messages,
|
||||||
|
max_completion_tokens=max_completion_tokens,
|
||||||
|
temperature=temperature,
|
||||||
|
)
|
||||||
|
return assert_non_empty_content(chat_completion, context=context)
|
||||||
|
|
||||||
|
|
||||||
def dummy_messages_from_video_url(
|
def dummy_messages_from_video_url(
|
||||||
video_urls: str | list[str],
|
video_urls: str | list[str],
|
||||||
content_text: str = "What's in this video?",
|
content_text: str = "What's in this video?",
|
||||||
@@ -402,3 +473,44 @@ async def test_multi_video_input(
|
|||||||
)
|
)
|
||||||
message = chat_completion.choices[0].message
|
message = chat_completion.choices[0].message
|
||||||
assert message.content is not None and len(message.content) >= 0
|
assert message.content is not None and len(message.content) >= 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||||
|
async def test_omitted_video_url_requires_cached_uuid(
|
||||||
|
client: openai.AsyncOpenAI,
|
||||||
|
model_name: str,
|
||||||
|
):
|
||||||
|
video_uuid = "test-video-uuid"
|
||||||
|
video_url = TEST_VIDEO_URLS[0]
|
||||||
|
|
||||||
|
omitted_video_messages = describe_video_messages(
|
||||||
|
None,
|
||||||
|
extra_video_fields={"uuid": video_uuid},
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
openai.BadRequestError,
|
||||||
|
match="Cache miss for video at index 0 but data is not provided",
|
||||||
|
):
|
||||||
|
await client.chat.completions.create(
|
||||||
|
model=model_name,
|
||||||
|
messages=omitted_video_messages,
|
||||||
|
)
|
||||||
|
|
||||||
|
await complete_and_check(
|
||||||
|
client,
|
||||||
|
model_name,
|
||||||
|
describe_video_messages(
|
||||||
|
video_url,
|
||||||
|
extra_video_fields={"uuid": video_uuid},
|
||||||
|
),
|
||||||
|
context="populate UUID cache",
|
||||||
|
)
|
||||||
|
|
||||||
|
await complete_and_check(
|
||||||
|
client,
|
||||||
|
model_name,
|
||||||
|
omitted_video_messages,
|
||||||
|
context="reuse cached video without URL",
|
||||||
|
)
|
||||||
|
|||||||
@@ -5,7 +5,11 @@ import pytest
|
|||||||
import torch
|
import torch
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
from vllm.multimodal.parse import ImageProcessorItems, VideoProcessorItems
|
from vllm.multimodal.parse import (
|
||||||
|
ImageProcessorItems,
|
||||||
|
MultiModalDataParser,
|
||||||
|
VideoProcessorItems,
|
||||||
|
)
|
||||||
|
|
||||||
H, W = 480, 640
|
H, W = 480, 640
|
||||||
|
|
||||||
@@ -49,3 +53,18 @@ def test_frame_size_hwc_chw(frame):
|
|||||||
items = VideoProcessorItems([[frame]])
|
items = VideoProcessorItems([[frame]])
|
||||||
|
|
||||||
assert items.get_frame_size(0) == (W, H)
|
assert items.get_frame_size(0) == (W, H)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"modality,processor_cls",
|
||||||
|
[
|
||||||
|
("image", ImageProcessorItems),
|
||||||
|
("video", VideoProcessorItems),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_parse_mm_data_accepts_none_cached_item(modality, processor_cls):
|
||||||
|
mm_items = MultiModalDataParser().parse_mm_data({modality: [None]})
|
||||||
|
items = mm_items[modality]
|
||||||
|
assert isinstance(items, processor_cls)
|
||||||
|
assert len(items) == 1
|
||||||
|
assert items.get(0) is None
|
||||||
|
|||||||
@@ -756,9 +756,16 @@ class MultiModalDataParser:
|
|||||||
np.ndarray
|
np.ndarray
|
||||||
| MediaWithBytes[np.ndarray]
|
| MediaWithBytes[np.ndarray]
|
||||||
| tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any]]
|
| tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any]]
|
||||||
|
| None
|
||||||
]()
|
]()
|
||||||
metadata_lst: list[dict[str, Any] | None] = []
|
metadata_lst: list[dict[str, Any] | None] = []
|
||||||
for data_item in data_items:
|
for data_item in data_items:
|
||||||
|
# Allow None video items, valid requests can contain empty URLs
|
||||||
|
# if they use multi-modal uuids.
|
||||||
|
if data_item is None:
|
||||||
|
new_videos.append(None)
|
||||||
|
metadata_lst.append(None)
|
||||||
|
continue
|
||||||
video, metadata = self._get_video_with_metadata(data_item)
|
video, metadata = self._get_video_with_metadata(data_item)
|
||||||
if self.video_needs_metadata:
|
if self.video_needs_metadata:
|
||||||
if metadata is None:
|
if metadata is None:
|
||||||
|
|||||||
Reference in New Issue
Block a user