[1/N] Harden Transformers modelling backend multi-modal path (#51408)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -56,24 +56,52 @@ def test_multimodal_processor(model_id):
|
||||
)
|
||||
|
||||
|
||||
def test_image_multiple_inputs():
|
||||
"""Multiple images per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers backend."""
|
||||
def _process_two_images(separator: str):
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
prompt = (
|
||||
"<|im_start|>user <image>\n and <image>\n"
|
||||
f"<|im_start|>user <image>{separator}<image>\n"
|
||||
"What do these images show?<|im_end|><|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
result = mm_processor(
|
||||
return mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
def test_image_multiple_inputs():
|
||||
"""Multiple images per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
result = _process_two_images(separator="\n and ")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
|
||||
|
||||
def test_image_adjacent_inputs():
|
||||
"""Adjacent images stay separate placeholders rather than merging into one."""
|
||||
result = _process_two_images(separator="")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
|
||||
|
||||
def test_text_only_prompt():
|
||||
"""An image model still accepts a prompt with no images."""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
result = mm_processor(
|
||||
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
|
||||
mm_items=mm_processor.info.parse_mm_data({}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert len(result["prompt_token_ids"]) > 0
|
||||
assert not result["mm_placeholders"]
|
||||
|
||||
Reference in New Issue
Block a user