[1/N] Harden Transformers modelling backend multi-modal path (#51408)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -134,6 +134,17 @@ def get_model_ids_to_test():
|
||||
return _get_model_ids_to_test(vllm_only_archs)
|
||||
|
||||
|
||||
def get_transformers_backend_model_ids_to_test():
|
||||
return sorted(
|
||||
{
|
||||
model_id
|
||||
for arch, info in _TRANSFORMERS_BACKEND_MODELS.items()
|
||||
if "MultiModal" in arch
|
||||
for model_id in (info.default, *info.extras.values())
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def get_text_token_prompts(
|
||||
processor: BaseMultiModalProcessor,
|
||||
mm_data: MultiModalDataDict,
|
||||
@@ -211,6 +222,7 @@ def _test_processing_correctness(
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
model_impl: str = "auto",
|
||||
):
|
||||
if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
|
||||
# Use model architecture to get the default model id
|
||||
@@ -238,6 +250,7 @@ def _test_processing_correctness(
|
||||
enable_mm_embeds=model_info.require_embed_inputs,
|
||||
enforce_eager=model_info.enforce_eager,
|
||||
dtype=model_info.dtype,
|
||||
model_impl=model_impl,
|
||||
)
|
||||
# Ensure that the cache can fit all of the data
|
||||
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
|
||||
@@ -480,6 +493,25 @@ def test_processing_correctness(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", get_transformers_backend_model_ids_to_test())
|
||||
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
|
||||
@pytest.mark.parametrize("num_batches", [32])
|
||||
@pytest.mark.parametrize("simplify_rate", [1.0])
|
||||
def test_processing_correctness_transformers(
|
||||
model_id: str,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
):
|
||||
_test_processing_correctness(
|
||||
model_id,
|
||||
hit_rate=hit_rate,
|
||||
num_batches=num_batches,
|
||||
simplify_rate=simplify_rate,
|
||||
model_impl="transformers",
|
||||
)
|
||||
|
||||
|
||||
def _assert_inputs_equal(
|
||||
a: MultiModalInput,
|
||||
b: MultiModalInput,
|
||||
|
||||
@@ -113,9 +113,7 @@ def test_audio_multimodal_processor(model_id):
|
||||
)
|
||||
|
||||
|
||||
def test_audio_multiple_inputs():
|
||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers backend."""
|
||||
def _process_granite_speech(separator: str):
|
||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
@@ -124,15 +122,54 @@ def test_audio_multiple_inputs():
|
||||
# One token per audio; the processor expands each to its placeholder run.
|
||||
prompt = (
|
||||
"<|start_of_role|>user<|end_of_role|>"
|
||||
f"{audio_token} and {audio_token} transcribe<|end_of_text|>\n"
|
||||
f"{audio_token}{separator}{audio_token} transcribe<|end_of_text|>\n"
|
||||
)
|
||||
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
|
||||
|
||||
result = mm_processor(
|
||||
return mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
def test_audio_multiple_inputs():
|
||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
result = _process_granite_speech(separator=" and ")
|
||||
|
||||
assert len(result["mm_placeholders"]["audio"]) == 2
|
||||
assert len(result["mm_kwargs"]["audio"]) == 2
|
||||
|
||||
|
||||
def test_audio_fields_not_claimed_by_image():
|
||||
"""Audio fields survive when the image branch is also active."""
|
||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
audio_keys = ["input_features", "input_features_mask"]
|
||||
owned = mm_processor._partition_keys_by_modality(audio_keys, ["audio", "image"])
|
||||
|
||||
assert owned["audio"] == audio_keys
|
||||
assert owned["image"] == []
|
||||
|
||||
|
||||
def test_unclaimed_fields_warn_rather_than_raise():
|
||||
"""Keys no sub-processor declares are dropped with a warning, not an error."""
|
||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
owned = mm_processor._partition_keys_by_modality(
|
||||
["input_features", "surprise_field"], ["audio", "image"]
|
||||
)
|
||||
|
||||
assert owned["audio"] == ["input_features"]
|
||||
assert owned["image"] == []
|
||||
|
||||
|
||||
def test_audio_adjacent_inputs():
|
||||
"""Adjacent audios are rejected rather than silently merged into one placeholder."""
|
||||
with pytest.raises(ValueError, match="told apart"):
|
||||
_process_granite_speech(separator="")
|
||||
|
||||
@@ -56,24 +56,52 @@ def test_multimodal_processor(model_id):
|
||||
)
|
||||
|
||||
|
||||
def test_image_multiple_inputs():
|
||||
"""Multiple images per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers backend."""
|
||||
def _process_two_images(separator: str):
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
prompt = (
|
||||
"<|im_start|>user <image>\n and <image>\n"
|
||||
f"<|im_start|>user <image>{separator}<image>\n"
|
||||
"What do these images show?<|im_end|><|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
result = mm_processor(
|
||||
return mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
def test_image_multiple_inputs():
|
||||
"""Multiple images per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
result = _process_two_images(separator="\n and ")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
|
||||
|
||||
def test_image_adjacent_inputs():
|
||||
"""Adjacent images stay separate placeholders rather than merging into one."""
|
||||
result = _process_two_images(separator="")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
|
||||
|
||||
def test_text_only_prompt():
|
||||
"""An image model still accepts a prompt with no images."""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
result = mm_processor(
|
||||
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
|
||||
mm_items=mm_processor.info.parse_mm_data({}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert len(result["prompt_token_ids"]) > 0
|
||||
assert not result["mm_placeholders"]
|
||||
|
||||
Reference in New Issue
Block a user