[Misc] Remove code paths for Transformers < 5.16.1 (#59762)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-02 11:17:24 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 8c60714da5
commit 5d20979fc9
18 changed files with 142 additions and 757 deletions
@@ -5,23 +5,16 @@ from unittest.mock import patch
import pytest
from vllm.assets.image import ImageAsset
from vllm.model_executor.models.transformers.multimodal import (
LegacyMultiModalProcessor,
OffsetsMultiModalProcessor,
)
from .transformers_backend import (
PROCESSOR_CLASSES,
create_cached_processor,
create_processor,
offsets_only,
)
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
def test_multimodal_processor(model_id, processor_cls):
mm_processor = create_processor(model_id, processor_cls)
def test_multimodal_processor(model_id):
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
image_pil = ImageAsset("cherry_blossom").pil_image
mm_data = {"image": image_pil}
@@ -63,9 +56,11 @@ def test_multimodal_processor(model_id, processor_cls):
)
def _process_two_images(processor_cls, separator: str):
def _process_two_images(separator: str):
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
image = ImageAsset("cherry_blossom").pil_image
prompt = (
@@ -80,30 +75,29 @@ def _process_two_images(processor_cls, separator: str):
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_image_multiple_inputs(processor_cls):
def test_image_multiple_inputs():
"""Multiple images per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
result = _process_two_images(processor_cls, separator="\n and ")
result = _process_two_images(separator="\n and ")
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_image_adjacent_inputs(processor_cls):
def test_image_adjacent_inputs():
"""Adjacent images stay separate placeholders rather than merging into one."""
result = _process_two_images(processor_cls, separator="")
result = _process_two_images(separator="")
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_batch_padding_removed_from_image_items(processor_cls):
def test_batch_padding_removed_from_image_items():
"""Emu3 pads every image up to the largest in the batch, which would leave an
item's data dependent on what it was processed with and so uncacheable."""
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model="BAAI/Emu3-Chat-hf", model_impl="transformers")
)
image_token = mm_processor.info.get_hf_processor().image_token
images = [
@@ -128,53 +122,39 @@ def test_batch_padding_removed_from_image_items(processor_cls):
assert len(shapes) == 2
def _process_one_gemma3_image(processor_cls):
mm_processor = create_processor("google/gemma-3-4b-it", processor_cls)
hf_processor = mm_processor.info.get_hf_processor()
result = mm_processor(
prompt=f"{hf_processor.boi_token} What is this?",
def _process_one_gemma3_image():
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model="google/gemma-3-4b-it", model_impl="transformers")
)
boi_token = mm_processor.info.get_hf_processor().boi_token
return mm_processor(
prompt=f"{boi_token} What is this?",
mm_items=mm_processor.info.parse_mm_data(
{"image": ImageAsset("cherry_blossom").pil_image}
),
hf_processor_mm_kwargs={},
)
return hf_processor, result
@offsets_only
def test_non_embedding_tokens_excluded_from_placeholders():
"""Gemma3 wraps each image in text that carries no embeddings, which must be
inside the placeholder range but masked out of it."""
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
result = _process_one_gemma3_image()
(placeholder,) = result["mm_placeholders"]["image"]
assert placeholder.is_embed is not None
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
def test_legacy_placeholders_hold_only_image_tokens():
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
marker in the unexpanded prompt, not the token the expansion repeats."""
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
(placeholder,) = result["mm_placeholders"]["image"]
assert placeholder.length == hf_processor.image_seq_length
prompt_ids = result["prompt_token_ids"]
covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length]
assert set(covered) == {hf_processor.tokenizer.image_token_id}
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
def test_tokens_structuring_an_image_are_masked_not_dropped():
"""SmolVLM splits each image into tiles introduced by tokens carrying no
embeddings. Those belong inside the placeholder and masked out, because the token
count the processor reports is over the whole span. Idefics3 also refuses a prompt
holding `<image>` when no images are passed, which is how the offsets path has to
tokenize it before splicing in the expansion."""
mm_processor = create_processor(
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
holding `<image>` when no images are passed, which is how the prompt has to be
tokenized before splicing in the expansion."""
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
result = mm_processor(
prompt="<image>What is this?",
@@ -189,12 +169,13 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
@offsets_only
def test_missing_replacement_offsets_names_the_processor():
"""A processor that reports no replacement offsets cannot be served, which must
be said plainly rather than surfacing later as a field config mismatch."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
hf_processor_cls = type(mm_processor.info.get_hf_processor())
hf_call = hf_processor_cls.__call__
@@ -216,11 +197,12 @@ def test_missing_replacement_offsets_names_the_processor():
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_text_only_prompt(processor_cls):
def test_text_only_prompt():
"""An image model still accepts a prompt with no images."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
result = mm_processor(
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
@@ -232,12 +214,14 @@ def test_text_only_prompt(processor_cls):
assert not result["mm_placeholders"]
@offsets_only
def test_repeated_image_hits_the_processor_cache():
"""Check that mm caching is actually working."""
mm_processor, cache = create_cached_processor(
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor
model_config = ModelConfig(
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", model_impl="transformers"
)
model_config.multimodal_config.mm_processor_cache_gb = 4
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
cache = MultiModalProcessorOnlyCache(model_config)
image = ImageAsset("cherry_blossom").pil_image
def process():
@@ -255,7 +239,6 @@ def test_repeated_image_hits_the_processor_cache():
assert first["mm_hashes"] == second["mm_hashes"]
@offsets_only
@pytest.mark.parametrize(
("model_id", "prompt"),
[
@@ -267,7 +250,7 @@ def test_repeated_image_hits_the_processor_cache():
"<image> and more text",
marks=pytest.mark.xfail(
reason="Emu3Processor prepends its BOS token only when images are "
"passed, so the unexpanded prompt the offsets path tokenizes never "
"passed, so the unexpanded prompt vLLM tokenizes never "
"gets one. Fixed by huggingface/transformers#47924, unreleased.",
strict=False,
),
@@ -275,33 +258,40 @@ def test_repeated_image_hits_the_processor_cache():
],
)
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
"""The offsets path splices the expansion into a prompt tokenized without any
multi-modal data, so its token ids have to come out the same as the ones the HF
processor produces itself, which is what the legacy path returns."""
prompt_ids = []
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
mm_processor = create_processor(model_id, processor_cls)
prompt_ids.append(
mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data(
{"image": ImageAsset("cherry_blossom").pil_image}
),
hf_processor_mm_kwargs={},
)["prompt_token_ids"]
)
"""The prompt is tokenized without any multi-modal data and the expansion spliced
in, so its token ids have to come out the same as the ones the HF processor
produces itself."""
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
info = mm_processor.info
image = ImageAsset("cherry_blossom").pil_image
legacy_ids, offsets_ids = prompt_ids
assert legacy_ids == offsets_ids
hf_processor = info.get_hf_processor()
prompt_ids = info.get_tokenizer().encode(
prompt, **info.default_tok_params.get_encode_kwargs()
)
hf_ids = info.ctx.call_hf_processor(
hf_processor,
dict(text=hf_processor.decode(prompt_ids), images=[image]),
dict(truncation=False, add_special_tokens=False),
)["input_ids"][0].tolist()
result = mm_processor(
prompt=prompt,
mm_items=info.parse_mm_data({"image": image}),
hf_processor_mm_kwargs={},
)
assert result["prompt_token_ids"] == hf_ids
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_nested_image_fields_split_per_image(processor_cls):
def test_nested_image_fields_split_per_image():
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
belonging to each image one dimension further in. Slicing the batch dimension
instead handed the first image every row and the second an empty tensor."""
mm_processor = create_processor(
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
image = ImageAsset("cherry_blossom").pil_image
result = mm_processor(