[3/N] Harden Transformers modelling backend multi-modal path (#51827)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-20 10:24:41 +00:00
committed by GitHub
parent 1eab6fef01
commit 6e85feb1b7
4 changed files with 838 additions and 161 deletions
@@ -4,8 +4,11 @@ import numpy as np
import pytest
from vllm.config import ModelConfig
from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor
from vllm.multimodal import MULTIMODAL_REGISTRY
from .transformers_backend import PROCESSOR_CLASSES, create_processor
AUDIO_MODEL_SETTINGS = {
"ibm-granite/granite-speech-3.3-2b": {
"prompt": (
@@ -53,6 +56,7 @@ AUDIO_MODEL_SETTINGS = {
}
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize(
"model_id",
[
@@ -61,12 +65,12 @@ AUDIO_MODEL_SETTINGS = {
pytest.param(
"mistralai/Voxtral-Mini-3B-2507",
marks=pytest.mark.xfail(
reason="MistralCommonBackend.encode does not produce the audio "
"placeholder token (ID 24) from raw text. apply_chat_template "
"yields token IDs with placeholders, but MultiModalProcessor."
"apply() decodes the prompt back to text and re-tokenizes, at "
"which point the placeholders are lost. Fix belongs in "
"mistral_common or in the Voxtral-specific path.",
reason="Voxtral's mistral_common processor does not compose with "
"the Transformers modelling backend. Loading it currently fails "
"outright, because MistralCommonBackend.from_pretrained rejects "
"the kwargs vLLM passes, and it implements no "
"`replace_audio_token`, so it would report no replacement "
"offsets. Both fixes belong in mistral_common or transformers.",
strict=False,
),
),
@@ -74,15 +78,10 @@ AUDIO_MODEL_SETTINGS = {
"zai-org/GLM-ASR-Nano-2512",
],
)
def test_audio_multimodal_processor(model_id):
def test_audio_multimodal_processor(model_id, processor_cls):
settings = AUDIO_MODEL_SETTINGS[model_id]
model_config = ModelConfig(
model=model_id,
model_impl="transformers",
)
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
mm_processor = create_processor(model_id, processor_cls)
audio = np.zeros(16000, dtype=np.float32)
mm_data = {"audio": (audio, 16000)}
@@ -113,10 +112,13 @@ def test_audio_multimodal_processor(model_id):
)
def _process_granite_speech(separator: str):
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize("separator", [" and ", ""])
def test_audio_multiple_inputs(separator, processor_cls):
"""Multiple audios per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
model_id = "ibm-granite/granite-speech-3.3-2b"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
mm_processor = create_processor(model_id, processor_cls)
audio_token = mm_processor.info.get_hf_processor().audio_token
# One token per audio; the processor expands each to its placeholder run.
@@ -126,17 +128,21 @@ def _process_granite_speech(separator: str):
)
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
return mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
def process():
return mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
# The legacy path reads placeholders off contiguous runs of the audio token, so
# it cannot tell adjacent ones apart and says so instead of merging them
if processor_cls is LegacyMultiModalProcessor and not separator:
with pytest.raises(ValueError, match="Separate them in the prompt"):
process()
return
def test_audio_multiple_inputs():
"""Multiple audios per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
result = _process_granite_speech(separator=" and ")
result = process()
assert len(result["mm_placeholders"]["audio"]) == 2
assert len(result["mm_kwargs"]["audio"]) == 2
@@ -167,9 +173,3 @@ def test_unclaimed_fields_warn_rather_than_raise():
assert owned["audio"] == ["input_features"]
assert owned["image"] == []
def test_audio_adjacent_inputs():
"""Adjacent audios are rejected rather than silently merged into one placeholder."""
with pytest.raises(ValueError, match="told apart"):
_process_granite_speech(separator="")
@@ -1,20 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import patch
import pytest
from vllm.assets.image import ImageAsset
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.model_executor.models.transformers.multimodal import (
LegacyMultiModalProcessor,
OffsetsMultiModalProcessor,
)
from .transformers_backend import (
PROCESSOR_CLASSES,
create_cached_processor,
create_processor,
offsets_only,
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
def test_multimodal_processor(model_id):
model_config = ModelConfig(
model=model_id,
model_impl="transformers",
)
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
def test_multimodal_processor(model_id, processor_cls):
mm_processor = create_processor(model_id, processor_cls)
image_pil = ImageAsset("cherry_blossom").pil_image
mm_data = {"image": image_pil}
@@ -56,10 +63,9 @@ def test_multimodal_processor(model_id):
)
def _process_two_images(separator: str):
def _process_two_images(processor_cls, separator: str):
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
mm_processor = create_processor(model_id, processor_cls)
image = ImageAsset("cherry_blossom").pil_image
prompt = (
@@ -74,28 +80,147 @@ def _process_two_images(separator: str):
)
def test_image_multiple_inputs():
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_image_multiple_inputs(processor_cls):
"""Multiple images per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
result = _process_two_images(separator="\n and ")
result = _process_two_images(processor_cls, separator="\n and ")
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
def test_image_adjacent_inputs():
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_image_adjacent_inputs(processor_cls):
"""Adjacent images stay separate placeholders rather than merging into one."""
result = _process_two_images(separator="")
result = _process_two_images(processor_cls, separator="")
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
def test_text_only_prompt():
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_batch_padding_removed_from_image_items(processor_cls):
"""Emu3 pads every image up to the largest in the batch, which would leave an
item's data dependent on what it was processed with and so uncacheable."""
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
image_token = mm_processor.info.get_hf_processor().image_token
images = [
ImageAsset("cherry_blossom").pil_image,
ImageAsset("cherry_blossom").pil_image.resize((256, 1024)),
]
result = mm_processor(
prompt=f"{image_token} and {image_token}",
mm_items=mm_processor.info.parse_mm_data({"image": images}),
hf_processor_mm_kwargs={},
)
items = result["mm_kwargs"]["image"]
shapes = set()
for item in items:
height, width = item["image_sizes"].data.flatten().tolist()
pixel_values = item["pixel_values"].data
assert tuple(pixel_values.shape[-2:]) == (height, width)
shapes.add(tuple(pixel_values.shape))
# Both images would have been padded to a common shape had they been kept
assert len(shapes) == 2
def _process_one_gemma3_image(processor_cls):
mm_processor = create_processor("google/gemma-3-4b-it", processor_cls)
hf_processor = mm_processor.info.get_hf_processor()
result = mm_processor(
prompt=f"{hf_processor.boi_token} What is this?",
mm_items=mm_processor.info.parse_mm_data(
{"image": ImageAsset("cherry_blossom").pil_image}
),
hf_processor_mm_kwargs={},
)
return hf_processor, result
@offsets_only
def test_non_embedding_tokens_excluded_from_placeholders():
"""Gemma3 wraps each image in text that carries no embeddings, which must be
inside the placeholder range but masked out of it."""
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
(placeholder,) = result["mm_placeholders"]["image"]
assert placeholder.is_embed is not None
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
def test_legacy_placeholders_hold_only_image_tokens():
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
marker in the unexpanded prompt, not the token the expansion repeats."""
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
(placeholder,) = result["mm_placeholders"]["image"]
assert placeholder.length == hf_processor.image_seq_length
prompt_ids = result["prompt_token_ids"]
covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length]
assert set(covered) == {hf_processor.tokenizer.image_token_id}
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
"""SmolVLM splits each image into tiles introduced by tokens carrying no
embeddings. Those belong inside the placeholder and masked out, because the token
count the processor reports is over the whole span. Idefics3 also refuses a prompt
holding `<image>` when no images are passed, which is how the offsets path has to
tokenize it before splicing in the expansion."""
mm_processor = create_processor(
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
)
result = mm_processor(
prompt="<image>What is this?",
mm_items=mm_processor.info.parse_mm_data(
{"image": ImageAsset("cherry_blossom").pil_image}
),
hf_processor_mm_kwargs={},
)
(placeholder,) = result["mm_placeholders"]["image"]
assert placeholder.is_embed is not None
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
@offsets_only
def test_missing_replacement_offsets_names_the_processor():
"""A processor that reports no replacement offsets cannot be served, which must
be said plainly rather than surfacing later as a field config mismatch."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
hf_processor_cls = type(mm_processor.info.get_hf_processor())
hf_call = hf_processor_cls.__call__
def without_offsets(self, *args, **kwargs):
hf_inputs = hf_call(self, *args, **kwargs)
hf_inputs.pop("text_replacement_offsets", None)
return hf_inputs
with (
patch.object(hf_processor_cls, "__call__", without_offsets),
pytest.raises(ValueError, match="LlavaOnevisionProcessor returned no"),
):
mm_processor(
prompt="<image>\nWhat is the content of this image?",
mm_items=mm_processor.info.parse_mm_data(
{"image": ImageAsset("cherry_blossom").pil_image}
),
hf_processor_mm_kwargs={},
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_text_only_prompt(processor_cls):
"""An image model still accepts a prompt with no images."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
mm_processor = create_processor(model_id, processor_cls)
result = mm_processor(
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
@@ -105,3 +230,87 @@ def test_text_only_prompt():
assert len(result["prompt_token_ids"]) > 0
assert not result["mm_placeholders"]
@offsets_only
def test_repeated_image_hits_the_processor_cache():
"""Check that mm caching is actually working."""
mm_processor, cache = create_cached_processor(
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor
)
image = ImageAsset("cherry_blossom").pil_image
def process():
return mm_processor(
prompt="<image>\nWhat is this?",
mm_items=mm_processor.info.parse_mm_data({"image": image}),
hf_processor_mm_kwargs={},
)
first, second = process(), process()
assert cache.make_stats().hits > 0
assert first["prompt_token_ids"] == second["prompt_token_ids"]
assert first["mm_hashes"] == second["mm_hashes"]
@offsets_only
@pytest.mark.parametrize(
("model_id", "prompt"),
[
("llava-hf/llava-onevision-qwen2-0.5b-ov-hf", "<image>\nWhat is this?"),
("google/gemma-3-4b-it", "<start_of_image> What is this?"),
("HuggingFaceTB/SmolVLM-256M-Instruct", "<image>What is this?"),
pytest.param(
"BAAI/Emu3-Chat-hf",
"<image> and more text",
marks=pytest.mark.xfail(
reason="Emu3Processor prepends its BOS token only when images are "
"passed, so the unexpanded prompt the offsets path tokenizes never "
"gets one. Fixed by huggingface/transformers#47924, unreleased.",
strict=False,
),
),
],
)
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
"""The offsets path splices the expansion into a prompt tokenized without any
multi-modal data, so its token ids have to come out the same as the ones the HF
processor produces itself, which is what the legacy path returns."""
prompt_ids = []
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
mm_processor = create_processor(model_id, processor_cls)
prompt_ids.append(
mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data(
{"image": ImageAsset("cherry_blossom").pil_image}
),
hf_processor_mm_kwargs={},
)["prompt_token_ids"]
)
legacy_ids, offsets_ids = prompt_ids
assert legacy_ids == offsets_ids
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_nested_image_fields_split_per_image(processor_cls):
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
belonging to each image one dimension further in. Slicing the batch dimension
instead handed the first image every row and the second an empty tensor."""
mm_processor = create_processor(
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
)
image = ImageAsset("cherry_blossom").pil_image
result = mm_processor(
prompt="<image> and <image>",
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
hf_processor_mm_kwargs={},
)
items = result["mm_kwargs"]["image"]
assert len(items) == 2
for item in items:
pixel_values = item["pixel_values"].data
assert pixel_values.shape[1] == int(item["num_image_patches"].data)
@@ -0,0 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helpers for testing both processors of the Transformers modelling backend."""
import pytest
from vllm.config import ModelConfig
from vllm.model_executor.models.transformers.multimodal import (
LegacyMultiModalProcessor,
MultiModalDummyInputsBuilder,
MultiModalProcessingInfo,
MultiModalProcessor,
OffsetsMultiModalProcessor,
)
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.processing import InputProcessingContext
from vllm.tokenizers.registry import cached_tokenizer_from_config
offsets_only = pytest.mark.skipif(
MultiModalProcessor is not OffsetsMultiModalProcessor,
reason="Replacement offsets are only used from transformers 5.15.0 onwards",
)
PROCESSOR_CLASSES = [
pytest.param(LegacyMultiModalProcessor, id="legacy"),
pytest.param(OffsetsMultiModalProcessor, id="offsets", marks=offsets_only),
]
def create_processor(model_id: str, processor_cls):
"""Build a processor directly, because the registry only ever builds the one the
installed transformers version selects, leaving the other path untested."""
model_config = ModelConfig(model=model_id, model_impl="transformers")
ctx = InputProcessingContext(
model_config, cached_tokenizer_from_config(model_config)
)
info = MultiModalProcessingInfo(ctx)
return processor_cls(info, MultiModalDummyInputsBuilder(info))
def create_cached_processor(model_id: str, processor_cls):
"""Build a processor backed by a real multi-modal processor cache, and hand the
cache back so a test can check it was actually used."""
model_config = ModelConfig(model=model_id, model_impl="transformers")
model_config.multimodal_config.mm_processor_cache_gb = 4
ctx = InputProcessingContext(
model_config, cached_tokenizer_from_config(model_config)
)
info = MultiModalProcessingInfo(ctx)
cache = MultiModalProcessorOnlyCache(model_config)
return processor_cls(info, MultiModalDummyInputsBuilder(info), cache=cache), cache