[3/N] Harden Transformers modelling backend multi-modal path (#51827)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -4,8 +4,11 @@ import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from .transformers_backend import PROCESSOR_CLASSES, create_processor
|
||||
|
||||
AUDIO_MODEL_SETTINGS = {
|
||||
"ibm-granite/granite-speech-3.3-2b": {
|
||||
"prompt": (
|
||||
@@ -53,6 +56,7 @@ AUDIO_MODEL_SETTINGS = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
@@ -61,12 +65,12 @@ AUDIO_MODEL_SETTINGS = {
|
||||
pytest.param(
|
||||
"mistralai/Voxtral-Mini-3B-2507",
|
||||
marks=pytest.mark.xfail(
|
||||
reason="MistralCommonBackend.encode does not produce the audio "
|
||||
"placeholder token (ID 24) from raw text. apply_chat_template "
|
||||
"yields token IDs with placeholders, but MultiModalProcessor."
|
||||
"apply() decodes the prompt back to text and re-tokenizes, at "
|
||||
"which point the placeholders are lost. Fix belongs in "
|
||||
"mistral_common or in the Voxtral-specific path.",
|
||||
reason="Voxtral's mistral_common processor does not compose with "
|
||||
"the Transformers modelling backend. Loading it currently fails "
|
||||
"outright, because MistralCommonBackend.from_pretrained rejects "
|
||||
"the kwargs vLLM passes, and it implements no "
|
||||
"`replace_audio_token`, so it would report no replacement "
|
||||
"offsets. Both fixes belong in mistral_common or transformers.",
|
||||
strict=False,
|
||||
),
|
||||
),
|
||||
@@ -74,15 +78,10 @@ AUDIO_MODEL_SETTINGS = {
|
||||
"zai-org/GLM-ASR-Nano-2512",
|
||||
],
|
||||
)
|
||||
def test_audio_multimodal_processor(model_id):
|
||||
def test_audio_multimodal_processor(model_id, processor_cls):
|
||||
settings = AUDIO_MODEL_SETTINGS[model_id]
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=model_id,
|
||||
model_impl="transformers",
|
||||
)
|
||||
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
|
||||
audio = np.zeros(16000, dtype=np.float32)
|
||||
mm_data = {"audio": (audio, 16000)}
|
||||
@@ -113,10 +112,13 @@ def test_audio_multimodal_processor(model_id):
|
||||
)
|
||||
|
||||
|
||||
def _process_granite_speech(separator: str):
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize("separator", [" and ", ""])
|
||||
def test_audio_multiple_inputs(separator, processor_cls):
|
||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
|
||||
audio_token = mm_processor.info.get_hf_processor().audio_token
|
||||
# One token per audio; the processor expands each to its placeholder run.
|
||||
@@ -126,17 +128,21 @@ def _process_granite_speech(separator: str):
|
||||
)
|
||||
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
|
||||
|
||||
return mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
def process():
|
||||
return mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
# The legacy path reads placeholders off contiguous runs of the audio token, so
|
||||
# it cannot tell adjacent ones apart and says so instead of merging them
|
||||
if processor_cls is LegacyMultiModalProcessor and not separator:
|
||||
with pytest.raises(ValueError, match="Separate them in the prompt"):
|
||||
process()
|
||||
return
|
||||
|
||||
def test_audio_multiple_inputs():
|
||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
result = _process_granite_speech(separator=" and ")
|
||||
result = process()
|
||||
|
||||
assert len(result["mm_placeholders"]["audio"]) == 2
|
||||
assert len(result["mm_kwargs"]["audio"]) == 2
|
||||
@@ -167,9 +173,3 @@ def test_unclaimed_fields_warn_rather_than_raise():
|
||||
|
||||
assert owned["audio"] == ["input_features"]
|
||||
assert owned["image"] == []
|
||||
|
||||
|
||||
def test_audio_adjacent_inputs():
|
||||
"""Adjacent audios are rejected rather than silently merged into one placeholder."""
|
||||
with pytest.raises(ValueError, match="told apart"):
|
||||
_process_granite_speech(separator="")
|
||||
|
||||
@@ -1,20 +1,27 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.model_executor.models.transformers.multimodal import (
|
||||
LegacyMultiModalProcessor,
|
||||
OffsetsMultiModalProcessor,
|
||||
)
|
||||
|
||||
from .transformers_backend import (
|
||||
PROCESSOR_CLASSES,
|
||||
create_cached_processor,
|
||||
create_processor,
|
||||
offsets_only,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||
def test_multimodal_processor(model_id):
|
||||
model_config = ModelConfig(
|
||||
model=model_id,
|
||||
model_impl="transformers",
|
||||
)
|
||||
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
def test_multimodal_processor(model_id, processor_cls):
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
|
||||
image_pil = ImageAsset("cherry_blossom").pil_image
|
||||
mm_data = {"image": image_pil}
|
||||
@@ -56,10 +63,9 @@ def test_multimodal_processor(model_id):
|
||||
)
|
||||
|
||||
|
||||
def _process_two_images(separator: str):
|
||||
def _process_two_images(processor_cls, separator: str):
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
prompt = (
|
||||
@@ -74,28 +80,147 @@ def _process_two_images(separator: str):
|
||||
)
|
||||
|
||||
|
||||
def test_image_multiple_inputs():
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_image_multiple_inputs(processor_cls):
|
||||
"""Multiple images per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
result = _process_two_images(separator="\n and ")
|
||||
result = _process_two_images(processor_cls, separator="\n and ")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
|
||||
|
||||
def test_image_adjacent_inputs():
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_image_adjacent_inputs(processor_cls):
|
||||
"""Adjacent images stay separate placeholders rather than merging into one."""
|
||||
result = _process_two_images(separator="")
|
||||
result = _process_two_images(processor_cls, separator="")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
|
||||
|
||||
def test_text_only_prompt():
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_batch_padding_removed_from_image_items(processor_cls):
|
||||
"""Emu3 pads every image up to the largest in the batch, which would leave an
|
||||
item's data dependent on what it was processed with and so uncacheable."""
|
||||
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
|
||||
image_token = mm_processor.info.get_hf_processor().image_token
|
||||
|
||||
images = [
|
||||
ImageAsset("cherry_blossom").pil_image,
|
||||
ImageAsset("cherry_blossom").pil_image.resize((256, 1024)),
|
||||
]
|
||||
result = mm_processor(
|
||||
prompt=f"{image_token} and {image_token}",
|
||||
mm_items=mm_processor.info.parse_mm_data({"image": images}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
items = result["mm_kwargs"]["image"]
|
||||
shapes = set()
|
||||
for item in items:
|
||||
height, width = item["image_sizes"].data.flatten().tolist()
|
||||
pixel_values = item["pixel_values"].data
|
||||
assert tuple(pixel_values.shape[-2:]) == (height, width)
|
||||
shapes.add(tuple(pixel_values.shape))
|
||||
|
||||
# Both images would have been padded to a common shape had they been kept
|
||||
assert len(shapes) == 2
|
||||
|
||||
|
||||
def _process_one_gemma3_image(processor_cls):
|
||||
mm_processor = create_processor("google/gemma-3-4b-it", processor_cls)
|
||||
hf_processor = mm_processor.info.get_hf_processor()
|
||||
result = mm_processor(
|
||||
prompt=f"{hf_processor.boi_token} What is this?",
|
||||
mm_items=mm_processor.info.parse_mm_data(
|
||||
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||
),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
return hf_processor, result
|
||||
|
||||
|
||||
@offsets_only
|
||||
def test_non_embedding_tokens_excluded_from_placeholders():
|
||||
"""Gemma3 wraps each image in text that carries no embeddings, which must be
|
||||
inside the placeholder range but masked out of it."""
|
||||
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
assert placeholder.is_embed is not None
|
||||
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
|
||||
|
||||
|
||||
def test_legacy_placeholders_hold_only_image_tokens():
|
||||
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
|
||||
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
|
||||
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
|
||||
marker in the unexpanded prompt, not the token the expansion repeats."""
|
||||
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
assert placeholder.length == hf_processor.image_seq_length
|
||||
prompt_ids = result["prompt_token_ids"]
|
||||
covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length]
|
||||
assert set(covered) == {hf_processor.tokenizer.image_token_id}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
|
||||
"""SmolVLM splits each image into tiles introduced by tokens carrying no
|
||||
embeddings. Those belong inside the placeholder and masked out, because the token
|
||||
count the processor reports is over the whole span. Idefics3 also refuses a prompt
|
||||
holding `<image>` when no images are passed, which is how the offsets path has to
|
||||
tokenize it before splicing in the expansion."""
|
||||
mm_processor = create_processor(
|
||||
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||
)
|
||||
result = mm_processor(
|
||||
prompt="<image>What is this?",
|
||||
mm_items=mm_processor.info.parse_mm_data(
|
||||
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||
),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
assert placeholder.is_embed is not None
|
||||
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
|
||||
|
||||
|
||||
@offsets_only
|
||||
def test_missing_replacement_offsets_names_the_processor():
|
||||
"""A processor that reports no replacement offsets cannot be served, which must
|
||||
be said plainly rather than surfacing later as a field config mismatch."""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
|
||||
hf_processor_cls = type(mm_processor.info.get_hf_processor())
|
||||
hf_call = hf_processor_cls.__call__
|
||||
|
||||
def without_offsets(self, *args, **kwargs):
|
||||
hf_inputs = hf_call(self, *args, **kwargs)
|
||||
hf_inputs.pop("text_replacement_offsets", None)
|
||||
return hf_inputs
|
||||
|
||||
with (
|
||||
patch.object(hf_processor_cls, "__call__", without_offsets),
|
||||
pytest.raises(ValueError, match="LlavaOnevisionProcessor returned no"),
|
||||
):
|
||||
mm_processor(
|
||||
prompt="<image>\nWhat is the content of this image?",
|
||||
mm_items=mm_processor.info.parse_mm_data(
|
||||
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||
),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_text_only_prompt(processor_cls):
|
||||
"""An image model still accepts a prompt with no images."""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
|
||||
result = mm_processor(
|
||||
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
|
||||
@@ -105,3 +230,87 @@ def test_text_only_prompt():
|
||||
|
||||
assert len(result["prompt_token_ids"]) > 0
|
||||
assert not result["mm_placeholders"]
|
||||
|
||||
|
||||
@offsets_only
|
||||
def test_repeated_image_hits_the_processor_cache():
|
||||
"""Check that mm caching is actually working."""
|
||||
mm_processor, cache = create_cached_processor(
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor
|
||||
)
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
|
||||
def process():
|
||||
return mm_processor(
|
||||
prompt="<image>\nWhat is this?",
|
||||
mm_items=mm_processor.info.parse_mm_data({"image": image}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
first, second = process(), process()
|
||||
|
||||
assert cache.make_stats().hits > 0
|
||||
assert first["prompt_token_ids"] == second["prompt_token_ids"]
|
||||
assert first["mm_hashes"] == second["mm_hashes"]
|
||||
|
||||
|
||||
@offsets_only
|
||||
@pytest.mark.parametrize(
|
||||
("model_id", "prompt"),
|
||||
[
|
||||
("llava-hf/llava-onevision-qwen2-0.5b-ov-hf", "<image>\nWhat is this?"),
|
||||
("google/gemma-3-4b-it", "<start_of_image> What is this?"),
|
||||
("HuggingFaceTB/SmolVLM-256M-Instruct", "<image>What is this?"),
|
||||
pytest.param(
|
||||
"BAAI/Emu3-Chat-hf",
|
||||
"<image> and more text",
|
||||
marks=pytest.mark.xfail(
|
||||
reason="Emu3Processor prepends its BOS token only when images are "
|
||||
"passed, so the unexpanded prompt the offsets path tokenizes never "
|
||||
"gets one. Fixed by huggingface/transformers#47924, unreleased.",
|
||||
strict=False,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
|
||||
"""The offsets path splices the expansion into a prompt tokenized without any
|
||||
multi-modal data, so its token ids have to come out the same as the ones the HF
|
||||
processor produces itself, which is what the legacy path returns."""
|
||||
prompt_ids = []
|
||||
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
prompt_ids.append(
|
||||
mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data(
|
||||
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||
),
|
||||
hf_processor_mm_kwargs={},
|
||||
)["prompt_token_ids"]
|
||||
)
|
||||
|
||||
legacy_ids, offsets_ids = prompt_ids
|
||||
assert legacy_ids == offsets_ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_nested_image_fields_split_per_image(processor_cls):
|
||||
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
|
||||
belonging to each image one dimension further in. Slicing the batch dimension
|
||||
instead handed the first image every row and the second an empty tensor."""
|
||||
mm_processor = create_processor(
|
||||
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||
)
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
result = mm_processor(
|
||||
prompt="<image> and <image>",
|
||||
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
items = result["mm_kwargs"]["image"]
|
||||
assert len(items) == 2
|
||||
for item in items:
|
||||
pixel_values = item["pixel_values"].data
|
||||
assert pixel_values.shape[1] == int(item["num_image_patches"].data)
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Helpers for testing both processors of the Transformers modelling backend."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.transformers.multimodal import (
|
||||
LegacyMultiModalProcessor,
|
||||
MultiModalDummyInputsBuilder,
|
||||
MultiModalProcessingInfo,
|
||||
MultiModalProcessor,
|
||||
OffsetsMultiModalProcessor,
|
||||
)
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.processing import InputProcessingContext
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
|
||||
offsets_only = pytest.mark.skipif(
|
||||
MultiModalProcessor is not OffsetsMultiModalProcessor,
|
||||
reason="Replacement offsets are only used from transformers 5.15.0 onwards",
|
||||
)
|
||||
|
||||
PROCESSOR_CLASSES = [
|
||||
pytest.param(LegacyMultiModalProcessor, id="legacy"),
|
||||
pytest.param(OffsetsMultiModalProcessor, id="offsets", marks=offsets_only),
|
||||
]
|
||||
|
||||
|
||||
def create_processor(model_id: str, processor_cls):
|
||||
"""Build a processor directly, because the registry only ever builds the one the
|
||||
installed transformers version selects, leaving the other path untested."""
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
ctx = InputProcessingContext(
|
||||
model_config, cached_tokenizer_from_config(model_config)
|
||||
)
|
||||
info = MultiModalProcessingInfo(ctx)
|
||||
return processor_cls(info, MultiModalDummyInputsBuilder(info))
|
||||
|
||||
|
||||
def create_cached_processor(model_id: str, processor_cls):
|
||||
"""Build a processor backed by a real multi-modal processor cache, and hand the
|
||||
cache back so a test can check it was actually used."""
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 4
|
||||
ctx = InputProcessingContext(
|
||||
model_config, cached_tokenizer_from_config(model_config)
|
||||
)
|
||||
info = MultiModalProcessingInfo(ctx)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
return processor_cls(info, MultiModalDummyInputsBuilder(info), cache=cache), cache
|
||||
Reference in New Issue
Block a user