[Bugfix] Raise clear error on interleaved multimodal placeholder overcount (#53999)
Signed-off-by: hungh <hungh@nvidia.com>
This commit is contained in:
@@ -1663,6 +1663,30 @@ def test_parse_chat_messages_multiple_images_interleave(
|
|||||||
_assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
|
_assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_chat_messages_interleave_placeholder_overcount_raises(
|
||||||
|
phi3v_model_config_mm_interleaved,
|
||||||
|
image_url,
|
||||||
|
):
|
||||||
|
# Only one image is supplied, but the user's text also contains the internal
|
||||||
|
# image sentinel, so the interleaved prompt references more image
|
||||||
|
# placeholders than actual images. This must surface as a clear validation
|
||||||
|
# error rather than an IndexError from popping an empty list.
|
||||||
|
with pytest.raises(VLLMValidationError, match="placeholders in input prompt"):
|
||||||
|
parse_chat_messages(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
{"type": "image_url", "image_url": {"url": image_url}},
|
||||||
|
{"type": "text", "text": "<##IMAGE##>"},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
phi3v_model_config_mm_interleaved,
|
||||||
|
content_format="string",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_parse_chat_messages_multiple_images_interleave_async(
|
async def test_parse_chat_messages_multiple_images_interleave_async(
|
||||||
phi3v_model_config_mm_interleaved,
|
phi3v_model_config_mm_interleaved,
|
||||||
|
|||||||
@@ -1407,7 +1407,13 @@ def _get_interleaved_text_prompt(
|
|||||||
) -> str:
|
) -> str:
|
||||||
for idx, elem in enumerate(texts):
|
for idx, elem in enumerate(texts):
|
||||||
if elem in placeholder_storage:
|
if elem in placeholder_storage:
|
||||||
texts[idx] = placeholder_storage[elem].pop(0)
|
slots = placeholder_storage[elem]
|
||||||
|
if not slots:
|
||||||
|
raise VLLMValidationError(
|
||||||
|
f"Found more '{elem}' placeholders in input prompt than "
|
||||||
|
"actual multimodal data items."
|
||||||
|
)
|
||||||
|
texts[idx] = slots.pop(0)
|
||||||
|
|
||||||
return "\n".join(texts)
|
return "\n".join(texts)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user