[Bugfix] Raise clear error on interleaved multimodal placeholder overcount (#53999)
Signed-off-by: hungh <hungh@nvidia.com>
This commit is contained in:
@@ -1663,6 +1663,30 @@ def test_parse_chat_messages_multiple_images_interleave(
|
||||
_assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
|
||||
|
||||
|
||||
def test_parse_chat_messages_interleave_placeholder_overcount_raises(
|
||||
phi3v_model_config_mm_interleaved,
|
||||
image_url,
|
||||
):
|
||||
# Only one image is supplied, but the user's text also contains the internal
|
||||
# image sentinel, so the interleaved prompt references more image
|
||||
# placeholders than actual images. This must surface as a clear validation
|
||||
# error rather than an IndexError from popping an empty list.
|
||||
with pytest.raises(VLLMValidationError, match="placeholders in input prompt"):
|
||||
parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": image_url}},
|
||||
{"type": "text", "text": "<##IMAGE##>"},
|
||||
],
|
||||
}
|
||||
],
|
||||
phi3v_model_config_mm_interleaved,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_chat_messages_multiple_images_interleave_async(
|
||||
phi3v_model_config_mm_interleaved,
|
||||
|
||||
@@ -1407,7 +1407,13 @@ def _get_interleaved_text_prompt(
|
||||
) -> str:
|
||||
for idx, elem in enumerate(texts):
|
||||
if elem in placeholder_storage:
|
||||
texts[idx] = placeholder_storage[elem].pop(0)
|
||||
slots = placeholder_storage[elem]
|
||||
if not slots:
|
||||
raise VLLMValidationError(
|
||||
f"Found more '{elem}' placeholders in input prompt than "
|
||||
"actual multimodal data items."
|
||||
)
|
||||
texts[idx] = slots.pop(0)
|
||||
|
||||
return "\n".join(texts)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user