[Misc] Remove code paths for Transformers < 5.16.1 (#59762)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-02 11:17:24 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 8c60714da5
commit 5d20979fc9
18 changed files with 142 additions and 757 deletions
@@ -5,16 +5,15 @@ from transformers import GraniteConfig
from vllm.model_executor.models.granite import granite_layer_attn_params
from ...utils import check_logprobs_close, check_transformers_version
from ...utils import check_logprobs_close
# model -> minimum transformers version, or None if unconstrained
MODELS = {
MODELS = [
# TODO(sang): Sliding window should be tested separately.
"ibm/PowerLM-3b": None,
"ibm/PowerMoE-3b": None,
"ibm-granite/granite-swash-2b": "5.15.1",
"ibm-granite/granite-swash-3b-a600m": "5.15.1",
}
"ibm/PowerLM-3b",
"ibm/PowerMoE-3b",
"ibm-granite/granite-swash-2b",
"ibm-granite/granite-swash-3b-a600m",
]
@pytest.mark.parametrize("model", MODELS)
@@ -31,8 +30,6 @@ def test_models(
max_tokens: int,
num_logprobs: int,
) -> None:
check_transformers_version(model, min_transformers_version=MODELS[model])
with hf_runner(model, dtype=dtype) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
@@ -4,11 +4,8 @@ import numpy as np
import pytest
from vllm.config import ModelConfig
from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor
from vllm.multimodal import MULTIMODAL_REGISTRY
from .transformers_backend import PROCESSOR_CLASSES, create_processor
AUDIO_MODEL_SETTINGS = {
"ibm-granite/granite-speech-3.3-2b": {
"prompt": (
@@ -56,7 +53,6 @@ AUDIO_MODEL_SETTINGS = {
}
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize(
"model_id",
[
@@ -78,10 +74,12 @@ AUDIO_MODEL_SETTINGS = {
"zai-org/GLM-ASR-Nano-2512",
],
)
def test_audio_multimodal_processor(model_id, processor_cls):
def test_audio_multimodal_processor(model_id):
settings = AUDIO_MODEL_SETTINGS[model_id]
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
audio = np.zeros(16000, dtype=np.float32)
mm_data = {"audio": (audio, 16000)}
@@ -112,13 +110,14 @@ def test_audio_multimodal_processor(model_id, processor_cls):
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize("separator", [" and ", ""])
def test_audio_multiple_inputs(separator, processor_cls):
def test_audio_multiple_inputs(separator):
"""Multiple audios per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
model_id = "ibm-granite/granite-speech-3.3-2b"
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
audio_token = mm_processor.info.get_hf_processor().audio_token
# One token per audio; the processor expands each to its placeholder run.
@@ -128,21 +127,11 @@ def test_audio_multiple_inputs(separator, processor_cls):
)
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
def process():
return mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
# The legacy path reads placeholders off contiguous runs of the audio token, so
# it cannot tell adjacent ones apart and says so instead of merging them
if processor_cls is LegacyMultiModalProcessor and not separator:
with pytest.raises(ValueError, match="Separate them in the prompt"):
process()
return
result = process()
result = mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
assert len(result["mm_placeholders"]["audio"]) == 2
assert len(result["mm_kwargs"]["audio"]) == 2
@@ -5,23 +5,16 @@ from unittest.mock import patch
import pytest
from vllm.assets.image import ImageAsset
from vllm.model_executor.models.transformers.multimodal import (
LegacyMultiModalProcessor,
OffsetsMultiModalProcessor,
)
from .transformers_backend import (
PROCESSOR_CLASSES,
create_cached_processor,
create_processor,
offsets_only,
)
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
def test_multimodal_processor(model_id, processor_cls):
mm_processor = create_processor(model_id, processor_cls)
def test_multimodal_processor(model_id):
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
image_pil = ImageAsset("cherry_blossom").pil_image
mm_data = {"image": image_pil}
@@ -63,9 +56,11 @@ def test_multimodal_processor(model_id, processor_cls):
)
def _process_two_images(processor_cls, separator: str):
def _process_two_images(separator: str):
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
image = ImageAsset("cherry_blossom").pil_image
prompt = (
@@ -80,30 +75,29 @@ def _process_two_images(processor_cls, separator: str):
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_image_multiple_inputs(processor_cls):
def test_image_multiple_inputs():
"""Multiple images per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
result = _process_two_images(processor_cls, separator="\n and ")
result = _process_two_images(separator="\n and ")
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_image_adjacent_inputs(processor_cls):
def test_image_adjacent_inputs():
"""Adjacent images stay separate placeholders rather than merging into one."""
result = _process_two_images(processor_cls, separator="")
result = _process_two_images(separator="")
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_batch_padding_removed_from_image_items(processor_cls):
def test_batch_padding_removed_from_image_items():
"""Emu3 pads every image up to the largest in the batch, which would leave an
item's data dependent on what it was processed with and so uncacheable."""
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model="BAAI/Emu3-Chat-hf", model_impl="transformers")
)
image_token = mm_processor.info.get_hf_processor().image_token
images = [
@@ -128,53 +122,39 @@ def test_batch_padding_removed_from_image_items(processor_cls):
assert len(shapes) == 2
def _process_one_gemma3_image(processor_cls):
mm_processor = create_processor("google/gemma-3-4b-it", processor_cls)
hf_processor = mm_processor.info.get_hf_processor()
result = mm_processor(
prompt=f"{hf_processor.boi_token} What is this?",
def _process_one_gemma3_image():
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model="google/gemma-3-4b-it", model_impl="transformers")
)
boi_token = mm_processor.info.get_hf_processor().boi_token
return mm_processor(
prompt=f"{boi_token} What is this?",
mm_items=mm_processor.info.parse_mm_data(
{"image": ImageAsset("cherry_blossom").pil_image}
),
hf_processor_mm_kwargs={},
)
return hf_processor, result
@offsets_only
def test_non_embedding_tokens_excluded_from_placeholders():
"""Gemma3 wraps each image in text that carries no embeddings, which must be
inside the placeholder range but masked out of it."""
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
result = _process_one_gemma3_image()
(placeholder,) = result["mm_placeholders"]["image"]
assert placeholder.is_embed is not None
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
def test_legacy_placeholders_hold_only_image_tokens():
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
marker in the unexpanded prompt, not the token the expansion repeats."""
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
(placeholder,) = result["mm_placeholders"]["image"]
assert placeholder.length == hf_processor.image_seq_length
prompt_ids = result["prompt_token_ids"]
covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length]
assert set(covered) == {hf_processor.tokenizer.image_token_id}
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
def test_tokens_structuring_an_image_are_masked_not_dropped():
"""SmolVLM splits each image into tiles introduced by tokens carrying no
embeddings. Those belong inside the placeholder and masked out, because the token
count the processor reports is over the whole span. Idefics3 also refuses a prompt
holding `<image>` when no images are passed, which is how the offsets path has to
tokenize it before splicing in the expansion."""
mm_processor = create_processor(
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
holding `<image>` when no images are passed, which is how the prompt has to be
tokenized before splicing in the expansion."""
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
result = mm_processor(
prompt="<image>What is this?",
@@ -189,12 +169,13 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
@offsets_only
def test_missing_replacement_offsets_names_the_processor():
"""A processor that reports no replacement offsets cannot be served, which must
be said plainly rather than surfacing later as a field config mismatch."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
hf_processor_cls = type(mm_processor.info.get_hf_processor())
hf_call = hf_processor_cls.__call__
@@ -216,11 +197,12 @@ def test_missing_replacement_offsets_names_the_processor():
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_text_only_prompt(processor_cls):
def test_text_only_prompt():
"""An image model still accepts a prompt with no images."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
result = mm_processor(
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
@@ -232,12 +214,14 @@ def test_text_only_prompt(processor_cls):
assert not result["mm_placeholders"]
@offsets_only
def test_repeated_image_hits_the_processor_cache():
"""Check that mm caching is actually working."""
mm_processor, cache = create_cached_processor(
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor
model_config = ModelConfig(
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", model_impl="transformers"
)
model_config.multimodal_config.mm_processor_cache_gb = 4
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
cache = MultiModalProcessorOnlyCache(model_config)
image = ImageAsset("cherry_blossom").pil_image
def process():
@@ -255,7 +239,6 @@ def test_repeated_image_hits_the_processor_cache():
assert first["mm_hashes"] == second["mm_hashes"]
@offsets_only
@pytest.mark.parametrize(
("model_id", "prompt"),
[
@@ -267,7 +250,7 @@ def test_repeated_image_hits_the_processor_cache():
"<image> and more text",
marks=pytest.mark.xfail(
reason="Emu3Processor prepends its BOS token only when images are "
"passed, so the unexpanded prompt the offsets path tokenizes never "
"passed, so the unexpanded prompt vLLM tokenizes never "
"gets one. Fixed by huggingface/transformers#47924, unreleased.",
strict=False,
),
@@ -275,33 +258,40 @@ def test_repeated_image_hits_the_processor_cache():
],
)
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
"""The offsets path splices the expansion into a prompt tokenized without any
multi-modal data, so its token ids have to come out the same as the ones the HF
processor produces itself, which is what the legacy path returns."""
prompt_ids = []
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
mm_processor = create_processor(model_id, processor_cls)
prompt_ids.append(
mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data(
{"image": ImageAsset("cherry_blossom").pil_image}
),
hf_processor_mm_kwargs={},
)["prompt_token_ids"]
)
"""The prompt is tokenized without any multi-modal data and the expansion spliced
in, so its token ids have to come out the same as the ones the HF processor
produces itself."""
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
info = mm_processor.info
image = ImageAsset("cherry_blossom").pil_image
legacy_ids, offsets_ids = prompt_ids
assert legacy_ids == offsets_ids
hf_processor = info.get_hf_processor()
prompt_ids = info.get_tokenizer().encode(
prompt, **info.default_tok_params.get_encode_kwargs()
)
hf_ids = info.ctx.call_hf_processor(
hf_processor,
dict(text=hf_processor.decode(prompt_ids), images=[image]),
dict(truncation=False, add_special_tokens=False),
)["input_ids"][0].tolist()
result = mm_processor(
prompt=prompt,
mm_items=info.parse_mm_data({"image": image}),
hf_processor_mm_kwargs={},
)
assert result["prompt_token_ids"] == hf_ids
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_nested_image_fields_split_per_image(processor_cls):
def test_nested_image_fields_split_per_image():
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
belonging to each image one dimension further in. Slicing the batch dimension
instead handed the first image every row and the second an empty tensor."""
mm_processor = create_processor(
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
image = ImageAsset("cherry_blossom").pil_image
result = mm_processor(
@@ -1,51 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helpers for testing both processors of the Transformers modelling backend."""
import pytest
from vllm.config import ModelConfig
from vllm.model_executor.models.transformers.multimodal import (
LegacyMultiModalProcessor,
MultiModalDummyInputsBuilder,
MultiModalProcessingInfo,
MultiModalProcessor,
OffsetsMultiModalProcessor,
)
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.processing import InputProcessingContext
from vllm.tokenizers.registry import cached_tokenizer_from_config
offsets_only = pytest.mark.skipif(
MultiModalProcessor is not OffsetsMultiModalProcessor,
reason="Replacement offsets are only used from transformers 5.15.0 onwards",
)
PROCESSOR_CLASSES = [
pytest.param(LegacyMultiModalProcessor, id="legacy"),
pytest.param(OffsetsMultiModalProcessor, id="offsets", marks=offsets_only),
]
def create_processor(model_id: str, processor_cls):
"""Build a processor directly, because the registry only ever builds the one the
installed transformers version selects, leaving the other path untested."""
model_config = ModelConfig(model=model_id, model_impl="transformers")
ctx = InputProcessingContext(
model_config, cached_tokenizer_from_config(model_config)
)
info = MultiModalProcessingInfo(ctx)
return processor_cls(info, MultiModalDummyInputsBuilder(info))
def create_cached_processor(model_id: str, processor_cls):
"""Build a processor backed by a real multi-modal processor cache, and hand the
cache back so a test can check it was actually used."""
model_config = ModelConfig(model=model_id, model_impl="transformers")
model_config.multimodal_config.mm_processor_cache_gb = 4
ctx = InputProcessingContext(
model_config, cached_tokenizer_from_config(model_config)
)
info = MultiModalProcessingInfo(ctx)
cache = MultiModalProcessorOnlyCache(model_config)
return processor_cls(info, MultiModalDummyInputsBuilder(info)), cache
+12 -74
View File
@@ -233,9 +233,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"CohereLabs/North-Mini-Code",
trust_remote_code=True,
is_available_online=False,
min_transformers_version="5.9.0",
),
"CwmForCausalLM": _HfExamplesInfo("facebook/cwm", min_transformers_version="4.58"),
"CwmForCausalLM": _HfExamplesInfo("facebook/cwm"),
# FIXME: databricks/dbrx-instruct has been deleted
"DbrxForCausalLM": _HfExamplesInfo(
"databricks/dbrx-instruct", is_available_online=False
@@ -266,9 +265,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True
),
"Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
"ExaoneMoeForCausalLM": _HfExamplesInfo(
"LGAI-EXAONE/K-EXAONE-236B-A23B", min_transformers_version="5.1.0"
),
"ExaoneMoeForCausalLM": _HfExamplesInfo("LGAI-EXAONE/K-EXAONE-236B-A23B"),
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
"FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
"FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
@@ -279,7 +276,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
"Gemma4ForCausalLM": _HfExamplesInfo(
"google/gemma-4-E2B-it",
min_transformers_version="5.0.0",
),
"Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
"GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
@@ -287,11 +283,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"),
"Glm4MoeLiteForCausalLM": _HfExamplesInfo(
"zai-org/GLM-4.7-Flash",
min_transformers_version="5.0.0",
),
"GlmMoeDsaForCausalLM": _HfExamplesInfo(
"zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False
),
"GlmMoeDsaForCausalLM": _HfExamplesInfo("zai-org/GLM-5", is_available_online=False),
"Glm5NextForCausalLM": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2"),
"GPTBigCodeForCausalLM": _HfExamplesInfo(
@@ -316,15 +309,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"GraniteMoeSharedForCausalLM": _HfExamplesInfo(
"ibm-research/moe-7b-1b-active-shared-experts"
),
"GraniteMoeSWAForCausalLM": _HfExamplesInfo(
"ibm-granite/granite-swash-3b-a600m", min_transformers_version="5.15.1"
),
"GraniteSWAForCausalLM": _HfExamplesInfo(
"ibm-granite/granite-swash-2b", min_transformers_version="5.15.1"
),
"GraniteMoeSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-3b-a600m"),
"GraniteSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-2b"),
"HrmTextForCausalLM": _HfExamplesInfo(
"sapientinc/HRM-Text-1B",
min_transformers_version="5.9.0",
),
"HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
"HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct"),
@@ -333,7 +321,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"HYV4ForCausalLM": _HfExamplesInfo("tencent/Hy4-preview", trust_remote_code=True),
"HyperCLOVAXForCausalLM": _HfExamplesInfo(
"naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
min_transformers_version="5.9.0",
),
"InklingForCausalLM": _HfExamplesInfo(
"thinkingmachines/Inkling-NVFP4",
@@ -353,9 +340,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"IQuestLoopCoderForCausalLM": _HfExamplesInfo(
"IQuestLab/IQuest-Coder-V1-40B-Loop-Instruct", trust_remote_code=True
),
"Jais2ForCausalLM": _HfExamplesInfo(
"inceptionai/Jais-2-8B-Chat", min_transformers_version="4.58"
),
"Jais2ForCausalLM": _HfExamplesInfo("inceptionai/Jais-2-8B-Chat"),
"JambaForCausalLM": _HfExamplesInfo(
"ai21labs/AI21-Jamba-1.5-Mini",
extras={
@@ -375,7 +360,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
"Lfm2MoeForCausalLM": _HfExamplesInfo(
"LiquidAI/LFM2-8B-A1B",
min_transformers_version="5.0.0",
use_original_num_layers=True,
# Initialize at least one MoE layer
hf_overrides={"num_hidden_layers": 4},
@@ -422,7 +406,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
),
"MiniCPM4ForCausalLM": _HfExamplesInfo(
"openbmb/MiniCPM4.1-8B",
min_transformers_version="4.56",
max_transformers_version="4.57",
transformers_version_reason={
"hf": "HF remote code imports removed `is_torch_fx_available`; "
@@ -523,7 +506,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Qwen3NextForCausalLM": _HfExamplesInfo(
"Qwen/Qwen3-Next-80B-A3B-Instruct",
extras={"tiny-random": "tiny-random/qwen3-next-moe"},
min_transformers_version="4.56.3",
),
"Rnj1ForCausalLM": _HfExamplesInfo(
"EssentialAI/rnj-1-instruct",
@@ -722,7 +704,6 @@ _TOKEN_CLASSIFICATION_EXAMPLE_MODELS = {
),
"OpenAIPrivacyFilterForTokenClassification": _HfExamplesInfo(
"openai/privacy-filter",
min_transformers_version="5.6.0.dev0",
),
"RobertaForTokenClassification": _HfExamplesInfo(
"Jean-Baptiste/roberta-large-ner-english"
@@ -794,7 +775,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
"AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo(
"nvidia/audio-flamingo-3-hf",
min_transformers_version="5.3.0",
transformers_version_reason={
"vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
},
@@ -823,22 +803,18 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
"CohereLabs/command-a-vision-07-2025",
extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"},
min_transformers_version="5.9.0",
),
"CohereCompassForConditionalGeneration": _HfExamplesInfo(
"CohereLabs/North-Micro-Vision-Instruct",
min_transformers_version="5.16.0",
),
"Cosmos3ForConditionalGeneration": _HfExamplesInfo(
"nvidia/Cosmos3-Nano",
extras={"super": "nvidia/Cosmos3-Super"},
max_model_len=4096,
min_transformers_version="4.57",
),
"Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo(
"nvidia/Cosmos3-Edge",
max_model_len=4096,
min_transformers_version="5.15",
use_original_num_layers=True,
hf_overrides={
"text_config": {
@@ -887,7 +863,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"Exaone4_5_ForConditionalGeneration": _HfExamplesInfo(
"LGAI-EXAONE/EXAONE-4.5-33B",
min_transformers_version="5.6.0",
),
"FireRedASR2ForConditionalGeneration": _HfExamplesInfo(
"allendou/FireRedASR2-LLM-vllm",
@@ -918,19 +893,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"Gemma4ForConditionalGeneration": _HfExamplesInfo(
"google/gemma-4-E2B-it",
min_transformers_version="5.5.0",
),
# TODO: update min_transformers_version when Gemma4 Unified lands in
# a stable transformers release.
"Gemma4UnifiedForConditionalGeneration": _HfExamplesInfo(
"google/gemma-4-12B-it",
min_transformers_version="5.8.0",
is_available_online=False,
),
"Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
"GlmAsrForConditionalGeneration": _HfExamplesInfo(
"zai-org/GLM-ASR-Nano-2512",
min_transformers_version="5.0.0",
),
"Granite4VisionForConditionalGeneration": _HfExamplesInfo(
"ibm-granite/granite-vision-4.1-4b",
@@ -943,7 +913,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"GraniteSpeechPlusForConditionalGeneration": _HfExamplesInfo(
"ibm-granite/granite-speech-4.1-2b-plus",
min_transformers_version="5.8.0",
),
"GLM4VForCausalLM": _HfExamplesInfo(
"zai-org/glm-4v-9b",
@@ -957,7 +926,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
"GlmOcrForConditionalGeneration": _HfExamplesInfo(
"zai-org/GLM-OCR",
min_transformers_version="5.1.0",
),
"Glm5NextForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
"H2OVLChatModel": _HfExamplesInfo(
@@ -974,7 +942,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"HunYuanVLForConditionalGeneration": _HfExamplesInfo(
"tencent/HunyuanOCR",
min_transformers_version="5.13",
),
"Idefics3ForConditionalGeneration": _HfExamplesInfo(
"HuggingFaceM4/Idefics3-8B-Llama3",
@@ -1087,7 +1054,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"Lfm2VlForConditionalGeneration": _HfExamplesInfo(
"LiquidAI/LFM2-VL-450M",
min_transformers_version="5.0.0",
),
"Llama4ForConditionalGeneration": _HfExamplesInfo(
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
@@ -1154,7 +1120,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
"openbmb/MiniCPM-V-4_6",
min_transformers_version="5.7.0",
),
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
"MiniMaxAI/MiniMax-M3",
@@ -1201,7 +1166,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Molmo2ForConditionalGeneration": _HfExamplesInfo(
"allenai/Molmo2-8B",
extras={"olmo": "allenai/Molmo2-O-7B"},
min_transformers_version="4.51",
trust_remote_code=True,
# required by current PrefixLM implementation
max_num_batched_tokens=31872,
@@ -1353,7 +1317,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"QianfanOCRForConditionalGeneration": _HfExamplesInfo(
"baidu/Qianfan-OCR",
min_transformers_version="5.6.0",
),
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen2-Audio-7B-Instruct"
@@ -1368,12 +1331,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen3VLForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-VL-4B-Instruct",
max_model_len=4096,
min_transformers_version="4.57",
),
"Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-VL-30B-A3B-Instruct",
max_model_len=4096,
min_transformers_version="4.57",
),
"Qwen3_5ForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3.5-0.8B",
@@ -1391,23 +1352,19 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
max_model_len=4096,
min_transformers_version="4.57",
),
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-ASR-0.6B",
max_model_len=4096,
min_transformers_version="4.57",
),
"Qwen3ASRRealtimeGeneration": _HfExamplesInfo(
"Qwen/Qwen3-ASR-0.6B",
max_model_len=4096,
min_transformers_version="4.57",
hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]},
),
"Qwen3ASRForcedAlignerForTokenClassification": _HfExamplesInfo(
"Qwen/Qwen3-ForcedAligner-0.6B",
max_model_len=4096,
min_transformers_version="4.57",
hf_overrides={"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]},
),
"RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
@@ -1456,7 +1413,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo(
"microsoft/VibeVoice-ASR-HF",
min_transformers_version="5.13.0",
),
"VoxtralForConditionalGeneration": _HfExamplesInfo(
"mistralai/Voxtral-Mini-3B-2507",
@@ -1530,7 +1486,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
use_original_num_layers=True, # DFlash requires all layers
max_model_len=8192, # Reduce for CI
max_num_seqs=32,
min_transformers_version="4.56.3", # Required for Qwen3Next
),
"MuseGlimmerAssistantModel": _HfExamplesInfo(
"meta-models/Muse-Glimmer-30B",
@@ -1576,7 +1531,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
speculative_model="Qwen/Qwen3-Omni-30B-A3B-DSpark",
is_available_online=False,
use_original_num_layers=True,
min_transformers_version="4.57",
),
"Gemma4DSparkModel": _HfExamplesInfo(
"google/gemma-4-12B-it",
@@ -1737,7 +1691,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"google/gemma-4-E4B-it",
speculative_model="google/gemma-4-E4B-it-assistant",
trust_remote_code=True,
min_transformers_version="5.8.0",
),
"ErnieMTPModel": _HfExamplesInfo(
"baidu/ERNIE-4.5-21B-A3B-PT",
@@ -1747,13 +1700,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"ExaoneMoeMTP": _HfExamplesInfo(
"LGAI-EXAONE/K-EXAONE-236B-A23B",
speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B",
min_transformers_version="5.1.0",
enable_prefix_caching=False,
),
"Exaone4_5_MTP": _HfExamplesInfo(
"LGAI-EXAONE/EXAONE-4.5-33B",
speculative_model="LGAI-EXAONE/EXAONE-4.5-33B",
min_transformers_version="5.6.0",
),
"ExtractHiddenStatesModel": _HfExamplesInfo(
"Qwen/Qwen3-8B",
@@ -1766,12 +1717,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"Glm4MoeLiteMTPModel": _HfExamplesInfo(
"zai-org/GLM-4.7-Flash",
speculative_model="zai-org/GLM-4.7-Flash",
min_transformers_version="5.0.0",
),
"GlmOcrMTPModel": _HfExamplesInfo(
"zai-org/GLM-OCR",
speculative_model="zai-org/GLM-OCR",
min_transformers_version="5.1.0",
),
"Glm5NextMTPModel": _HfExamplesInfo(
"zai-org/GLM-5.3-Flash",
@@ -1842,9 +1791,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
trust_remote_code=True,
is_available_online=False,
),
"Qwen3NextMTP": _HfExamplesInfo(
"Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3"
),
"Qwen3NextMTP": _HfExamplesInfo("Qwen/Qwen3-Next-80B-A3B-Instruct"),
"Qwen3_5MTP": _HfExamplesInfo(
"Qwen/Qwen3.5-0.8B",
speculative_model="Qwen/Qwen3.5-0.8B",
@@ -1865,29 +1812,20 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
}
_TRANSFORMERS_BACKEND_MODELS = {
"TransformersEmbeddingModel": _HfExamplesInfo(
"BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0"
),
"TransformersEmbeddingModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
"TransformersForSequenceClassification": _HfExamplesInfo(
"papluca/xlm-roberta-base-language-detection",
min_transformers_version="5.0.0",
),
"TransformersForCausalLM": _HfExamplesInfo(
"hmellor/Ilama-3.2-1B", trust_remote_code=True
),
"TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
"TransformersMoEForCausalLM": _HfExamplesInfo(
"allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0"
),
"TransformersMoEForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924"),
"TransformersMultiModalMoEForCausalLM": _HfExamplesInfo(
"Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0"
),
"TransformersMoEEmbeddingModel": _HfExamplesInfo(
"Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
),
"TransformersMoEForSequenceClassification": _HfExamplesInfo(
"Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
"Qwen/Qwen3-VL-30B-A3B-Instruct"
),
"TransformersMoEEmbeddingModel": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
"TransformersMoEForSequenceClassification": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
"TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"),
"TransformersMultiModalForSequenceClassification": _HfExamplesInfo(
"google/gemma-3-4b-it"
-11
View File
@@ -185,17 +185,6 @@ def test_sinks(hf_runner: type[HfRunner], vllm_runner: type[VllmRunner]) -> None
def test_mla(vllm_runner: type[VllmRunner], example_prompts: list[str]) -> None:
import transformers
from packaging.version import Version
installed = Version(transformers.__version__)
required = Version("5.15.0.dev0")
if installed < required:
pytest.skip(
"MLA models with the Transformers modeling backend require "
f"transformers>={required}, but got {installed}"
)
model = "hmellor/tiny-random-DeepseekV2ForCausalLM"
args = (example_prompts, 32, 5)
kwargs: dict[str, Any] = {"max_model_len": 2048, "enforce_eager": True}
-14
View File
@@ -609,17 +609,3 @@ def dummy_hf_overrides(
)
return hf_config
def check_transformers_version(
model: str,
min_transformers_version: str | None = None,
max_transformers_version: str | None = None,
):
from .registry import _HfExamplesInfo
return _HfExamplesInfo(
model,
min_transformers_version=min_transformers_version,
max_transformers_version=max_transformers_version,
).check_transformers_version(on_fail="skip")
-49
View File
@@ -10,7 +10,6 @@ from transformers import AutoConfig
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.hf import CachedHfTokenizer
from vllm.tokenizers.registry import (
TokenizerRegistry,
cached_get_tokenizer,
@@ -93,54 +92,6 @@ def test_resolve_tokenizer_args_rejects_hf_mistral_format_true():
)
@pytest.mark.parametrize(
("transformers_version", "model_type", "native_filename", "expect_error"),
[
pytest.param("5.14.0", "mistral", "tekken.json", True),
pytest.param("5.15.0", "mistral", "tekken.json", False),
pytest.param("5.14.0", "mistral3", "tekken.json", True),
pytest.param("5.15.0", "mistral3", "tekken.json", False),
pytest.param("5.14.0", "mixtral", "tokenizer.model.v1", True),
pytest.param("5.15.0", "mixtral", "tokenizer.model.v1", False),
pytest.param("5.14.0", "mistral", None, False),
pytest.param("5.14.0", "qwen3_5_moe", "tekken.json", False),
],
)
def test_get_tokenizer_handles_hf_mistral_transformers_compatibility(
tmp_path: Path,
transformers_version: str,
model_type: str,
native_filename: str | None,
expect_error: bool,
):
if native_filename is not None:
(tmp_path / native_filename).touch()
with (
patch(
"vllm.tokenizers.registry.get_config",
return_value=SimpleNamespace(model_type=model_type),
),
patch(
"vllm.tokenizers.registry.transformers.__version__",
transformers_version,
),
patch.object(
CachedHfTokenizer,
"from_pretrained",
return_value=SimpleNamespace(is_fast=True),
) as from_pretrained,
):
if expect_error:
with pytest.raises(ValueError, match="requires transformers>=5.15.0"):
get_tokenizer(str(tmp_path), tokenizer_mode="hf")
else:
tokenizer = get_tokenizer(str(tmp_path), tokenizer_mode="hf")
assert tokenizer.is_fast is True
assert from_pretrained.called == (not expect_error)
def test_customized_tokenizer():
TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__)
@@ -48,16 +48,6 @@ def check_mtp_correctness(
elif "gemma-4" in model_name:
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0}
if draft_model is not None and "gemma-4" in draft_model:
import transformers
from packaging.version import Version
if Version(transformers.__version__) < Version("5.8.0"):
pytest.skip(
"Gemma4 MTP assistant requires transformers>=5.8.0, "
f"got {transformers.__version__}"
)
with vllm_runner(
model_name,
block_size=None,
-13
View File
@@ -7,8 +7,6 @@ from typing import Annotated, Any, Literal, cast
import torch
import torch.nn as nn
import transformers
from packaging.version import Version
from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig
from transformers.models.pixtral import PixtralProcessor
@@ -59,10 +57,6 @@ from .utils import (
maybe_prefix,
)
TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY = Version(transformers.__version__) >= Version(
"5.15.0"
)
class Mistral3ImagePixelInputs(TensorSchema):
"""Dimensions:
@@ -284,13 +278,6 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
# PixtralProcessor supports image-only calls starting in transformers 5.15.
# Older releases need dummy text and take the slower tokenization path.
if not TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY:
return self.dummy_inputs.get_dummy_text(mm_counts)
return None
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
@@ -691,10 +691,8 @@ class Base(
# vLLM does not support encoder-decoder models, so if any encoder layer is
# found in a text only model, we assume the whole model is an encoder model
if has_encoder(self.model) and not is_multimodal(self.config):
self.check_version("5.0.0", "encoder models support")
return EncoderOnlyAttention
if self.model_config.use_mla:
self.check_version("5.15.0.dev0", "optimized MLA support")
if any(
isinstance(fuser, MLAFuser)
for fusers in self.fusers.values()
@@ -820,7 +818,6 @@ class Base(
)
def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
self.check_version("5.2.0", "Eagle3 support")
from transformers.utils.output_capturing import (
OutputRecorder,
maybe_install_capturing_hooks,
@@ -156,7 +156,6 @@ class MoEMixin(MixtureOfExperts, Base):
num_local_physical_experts: int
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
self.check_version("5.0.0", "MoE models support")
# Skip MixtureOfExperts.__init__ and call the next class in MRO
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
@@ -23,17 +23,11 @@ from typing import TYPE_CHECKING, Any
import torch
import transformers
from packaging.version import Version
from transformers.utils.generic import ModelOutput
from vllm.compilation.decorators import should_torch_compile_mm_encoder
from vllm.config.utils import getattr_iter
from vllm.inputs import (
MultiModalDataBuiltins,
MultiModalDataDict,
MultiModalInput,
mm_input,
)
from vllm.inputs import MultiModalDataBuiltins, MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
@@ -42,26 +36,15 @@ from vllm.model_executor.models.interfaces import (
)
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
PlaceholderRange,
)
from vllm.multimodal.parse import (
ImageProcessorItems,
ImageSize,
MultiModalDataItems,
MultiModalDataParser,
)
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig
from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
TimingContext,
cached_encode,
)
from vllm.sequence import IntermediateTensors
@@ -219,11 +202,13 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
return data
class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]):
"""Processing common to both Transformers backend processors: calling the HF
processor, sizing images, and attributing its outputs to a modality.
class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
"""Locates placeholders from the `text_replacement_offsets` the HF processor
reports, expressing each one as a `PromptUpdate`.
Subclasses add the strategy for locating placeholders in the prompt.
Stating the expansion as an update is what lets it be rebuilt from an
unexpanded prompt, so this processor takes the base class's processing path
and with it the multi-modal processor cache.
"""
def _get_modality_field_names(self, modality: str) -> set[str]:
@@ -415,301 +400,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
for image, (height, width) in zip(pixel_values, image_sizes.tolist())
]
class LegacyMultiModalProcessor(_MultiModalProcessorBase):
"""Locates placeholders by searching the prompt the HF processor has already
expanded for the tokens of each modality.
Serves transformers versions with no `return_text_replacement_offsets`.
Placeholders found this way cannot be rebuilt from an unexpanded prompt, so
this processor overrides `apply` and gets no multi-modal processor cache.
Remove it once `requirements/common.txt` requires `transformers>=5.15.0`.
"""
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
"""Empty, because `apply` writes the placeholder ranges itself rather than
deriving them from updates."""
return []
def _get_mm_token_ids(self, modality: str) -> list[int]:
"""Token ids marking where `modality` sits in the prompt, which for some
processors differ from the placeholder written into it.
The expanded prompt is all this path has to go on, so it takes the
processor at its word about which tokens belong to the modality.
"""
info = self.info
processor = info.get_hf_processor()
declared = getattr(processor, f"{modality}_token_ids", None) or ()
if ids := [token_id for token_id in declared if token_id is not None]:
return ids
config = info.get_hf_config()
names = (f"{modality}_token_id", f"{modality}_token_index")
token_id = getattr(processor, names[0], getattr_iter(config, names))
if token_id is None:
token = getattr(processor, f"{modality}_token", None)
if token is not None:
token_id = info.get_tokenizer().get_vocab().get(token)
if token_id is None:
raise ValueError(
f"Cannot find {modality}_token_id on processor or model config"
)
return [token_id]
def _apply_audio(
self,
prompt_ids: list[int],
processed_data: "BatchFeature",
num_audios: int,
) -> dict[str, list[PlaceholderRange]]:
"""Take each contiguous run of the audio token as one item's placeholder,
and record how many tokens the run holds."""
audio_token_ids = self._get_mm_token_ids("audio")
prompt_tensor = torch.tensor(prompt_ids)
is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
if not is_audio.any():
raise ValueError(
f"{num_audios} audio item(s) were passed but the prompt "
"contains no audio token. Add one placeholder per audio item."
)
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
transitions = padded.int().diff()
offsets = torch.where(transitions == 1)[0]
lengths = torch.where(transitions == -1)[0] - offsets
if len(offsets) != num_audios:
raise ValueError(
f"Found {len(offsets)} run(s) of the audio token in the prompt but "
f"{num_audios} audio item(s) were passed. The Transformers backend "
"locates audio placeholders by finding contiguous runs of the audio "
"token, so placeholders with no text between them cannot yet be told "
"apart. Separate them in the prompt to work around this."
)
ranges = [
PlaceholderRange(offset=offset.item(), length=length.item())
for offset, length in zip(offsets, lengths)
]
processed_data["num_audio_tokens"] = lengths
return {"audio": ranges}
def _get_num_multimodal_tokens(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, list[int]]:
"""Ask the HF processor how many tokens and patches each image expands to."""
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [
(size.height, size.width)
for size in map(images.get_image_size, range(len(images)))
]
return processor._get_num_multimodal_tokens(
image_sizes=image_sizes,
**self.info.ctx.get_merged_mm_kwargs({}),
)
def _apply_vision(
self,
prompt_ids: list[int],
processed_data: "BatchFeature",
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
mm_token_type_ids: torch.Tensor | None,
) -> dict[str, list[PlaceholderRange]]:
"""Split the positions the processor marks as image into one placeholder per
item, sized by the token count it reports for each image."""
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
if mm_token_type_ids is None:
raise ValueError(
f"{type(hf_processor).__name__} returned no `mm_token_type_ids`, so "
"the Transformers modeling backend cannot locate the placeholder of "
"each image."
)
# These mark the tokens structuring an image as well as the image itself,
# which is what the counts below are over
mm_positions = torch.where(mm_token_type_ids[0] == 1)[0]
mm_tokens_per_modality = self._get_num_multimodal_tokens(
mm_items, hf_processor_mm_kwargs
)
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
split_sizes = mm_tokens_per_modality["num_image_tokens"]
if sum(split_sizes) != len(mm_positions):
raise ValueError(
f"The expanded prompt holds {len(mm_positions)} image token(s) but "
f"{type(hf_processor).__name__} accounts for {sum(split_sizes)} "
f"across {mm_items.get_count('image')} image item(s)."
)
if split_sizes:
image_token_ids = torch.tensor(self._get_mm_token_ids("image"))
mm_tokens = torch.tensor(prompt_ids)[mm_positions]
ranges = [
PlaceholderRange(
offset=positions[0].item(),
length=positions.shape[0],
# Only some of the span carries embeddings
is_embed=torch.isin(tokens, image_token_ids),
)
for positions, tokens in zip(
torch.split(mm_positions, split_sizes),
torch.split(mm_tokens, split_sizes),
)
]
mm_placeholders = {"image": ranges}
processed_data["num_image_patches"] = torch.tensor(
mm_tokens_per_modality["num_image_patches"]
)
return mm_placeholders
def apply(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalInput:
"""Process the prompt and every multi-modal item in one HF processor call,
then read the placeholder ranges out of the token ids it returns."""
prompt = inputs.prompt
mm_items = inputs.mm_data_items
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
with timing_ctx.record("apply_hf_processor"):
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
prompt_text = hf_processor.decode(prompt)
# Bypass cached processor and always apply to the full set of mm inputs
# NOTE: we can't just set caching=False because base class method
# transforms outputs to `MultiModalKwargs` which is not going to
# work for Transformers. The vision path has logic tied to
# `mm_tokens_per_modality` in _apply_vision()
hf_processor_mm_kwargs = {
# vLLM needs the untruncated sequence to keep placeholder
# tokens aligned. Note that the text inputs are just dummy
# text, not the original prompt. The original prompt is
# already tokenized by the renderer.
"truncation": False,
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
# (the HF processor call disables `add_special_tokens`).
"add_special_tokens": False,
**hf_processor_mm_kwargs,
}
processor_data, _, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_processor_mm_kwargs
)
# The real prompt is fed to the processor below and the placeholder
# ranges are read back out of its output, so the dummy text meant for
# media-only callers is discarded here.
processor_data.pop("text", None)
has_mm_data = any(processor_data.values())
# Ask which modality owns each token of the expanded prompt, which
# is the only marking of the tokens an expansion adds around an item
if has_mm_data:
processor_data = {**processor_data, "return_mm_token_type_ids": True}
try:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **processor_data),
hf_processor_mm_kwargs,
)
except ValueError:
if has_mm_data:
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
prompt_ids = self.info.get_tokenizer().encode(prompt_text)
processed_data = transformers.BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
self._unpad_images(processed_data)
self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs)
processed_data.update(passthrough_data)
input_ids = processed_data.pop("input_ids")
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
# Use overrides if provided; fallback to data-dependent hashing.
with timing_ctx.record("get_mm_hashes"):
mm_hashes = inputs.get_mm_hashes(
self.info.model_id,
self.info.ctx.get_mm_config().mm_hasher_algorithm,
)
# Gemma3 reports them under the key the model uses for its own token types
mm_token_type_ids = processed_data.pop("token_type_ids", None)
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
if num_audios := mm_items.get_count("audio", strict=False):
mm_placeholders.update(
self._apply_audio(prompt_ids, processed_data, num_audios)
)
if mm_items.get_count("image", strict=False):
mm_placeholders.update(
self._apply_vision(
prompt_ids,
processed_data,
mm_items,
hf_processor_mm_kwargs,
mm_token_type_ids,
)
)
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
processed_data,
self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
)
# Bypassing `_maybe_apply_prompt_updates` also bypasses its validation.
# `_validate_mm_placeholders` can't be reused because it is typed for the
# `PlaceholderFeaturesInfo` the prompt update machinery produces.
mm_item_counts = mm_items.get_all_counts()
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
for modality, item_count in mm_item_counts.items():
num_placeholders = len(mm_placeholders.get(modality, []))
if num_placeholders != item_count:
raise RuntimeError(
f"Expected there to be {item_count} prompt placeholders "
f"corresponding to {item_count} {modality} items, but instead "
f"found {num_placeholders} prompt placeholders! Make sure the "
"prompt contains a placeholder token for each item."
)
return mm_input(
prompt_token_ids=prompt_ids,
mm_kwargs=mm_kwargs,
mm_hashes=mm_hashes,
mm_placeholders=mm_placeholders,
)
class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"""Locates placeholders from the `text_replacement_offsets` the HF processor
reports, expressing each one as a `PromptUpdate`.
Stating the expansion as an update is what lets it be rebuilt from an
unexpanded prompt, so this processor takes the base class's processing path
and with it the multi-modal processor cache.
"""
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
@@ -800,9 +490,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
except (AttributeError, KeyError, TypeError):
return None
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
@@ -856,8 +543,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"to reach `ProcessorMixin.get_text_with_replacements` with one "
"replacement per item, which usually means implementing "
"`replace_<modality>_token`. Please report this to transformers "
"so it can be fixed, and install `transformers<5.15.0` in the "
"meantime to locate placeholders in the expanded prompt instead."
"so it can be fixed."
)
hf_inputs.pop("input_ids", None)
return self._finalize_hf_mm_data(
@@ -896,15 +582,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
)
# From this version on, a processor reporting no offsets is an error rather than a
# fallback to searching the expanded prompt
MultiModalProcessor = (
LegacyMultiModalProcessor
if Version(transformers.__version__) < Version("5.15.0")
else OffsetsMultiModalProcessor
)
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
# Skip SupportsMRoPE.__init__ and call the next class in MRO
@@ -982,7 +659,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
super()._decorate_for_torch_compile()
# Decorate the encoder model classes to support torch compile if needed
if self.compilation_config.compile_mm_encoder:
self.check_version("5.0.0", "multimodal encoder compilation support")
encoder_classes = self._pre_trained_model_classes.encoders
if not encoder_classes:
raise ValueError(
@@ -1113,7 +789,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
if input_features is None:
return None
self.check_version("5.13.0", "audio models support")
num_audio_tokens = kwargs.pop("num_audio_tokens")
kwargs.pop("token_type_ids", None)
kwargs.pop("mm_token_type_ids", None)
+12 -15
View File
@@ -48,21 +48,18 @@ def detect_token_suffix(tokenizer: TokenizerLike) -> str:
round-trips.
"""
import transformers
if int(transformers.__version__.split(".")[0]) >= 5:
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
think_begin_as_special = init_kwargs.get(
"model_specific_special_tokens", {}
).get("think_begin_token", "")
if think_begin_as_special:
raise RuntimeError(
"This checkpoint declares HYV4 structural tokens (think_begin_token"
"/toolcalls_begin_token/argkey_begin_token) in "
"tokenizer_config.json, which transformers 5 no longer supports. "
"Remove those fields and keep the tokens in the tokenizer's own "
"token definitions so the suffix can be read from the vocab."
)
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
think_begin_as_special = init_kwargs.get("model_specific_special_tokens", {}).get(
"think_begin_token", ""
)
if think_begin_as_special:
raise RuntimeError(
"This checkpoint declares HYV4 structural tokens (think_begin_token"
"/toolcalls_begin_token/argkey_begin_token) in "
"tokenizer_config.json, which transformers 5 no longer supports. "
"Remove those fields and keep the tokens in the tokenizer's own "
"token definitions so the suffix can be read from the vocab."
)
structural_token_re = re.compile(
r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>"
-28
View File
@@ -7,12 +7,6 @@ from pathlib import Path
from typing import TYPE_CHECKING
import huggingface_hub
import transformers
from packaging.version import Version
from transformers.models.auto.tokenization_auto import (
TOKENIZER_MAPPING_NAMES as _HF_TOKENIZER_MAPPING_NAMES,
)
from transformers.utils.import_utils import is_mistral_common_available
from typing_extensions import TypeVar, assert_never
from vllm import envs
@@ -32,8 +26,6 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
_MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL = Version("5.15.0")
# Model types whose hub tokenizer_class is incorrect and should be overridden with
# TokenizersBackend (the generic fast tokenizer). Adding a model type here is always a
@@ -243,26 +235,6 @@ def get_tokenizer(
token=kwargs.get("token"),
)
# TODO: delete when Transformers version dependency is bumped >= 5.15.0
if (
tokenizer_mode == "hf"
and Version(transformers.__version__) < _MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL
and is_mistral_common_available()
and _HF_TOKENIZER_MAPPING_NAMES.get(getattr(config, "model_type", None))
== "MistralCommonBackend"
and any_pattern_in_repo_files(
model_name_or_path=str(tokenizer_name),
allow_patterns=["tekken.json", "tokenizer.model.v*"],
revision=revision,
token=kwargs.get("token"),
)
):
raise ValueError(
"Loading Mistral models with tokenizer_mode='hf' requires "
f"transformers>={_MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL}. "
"Please upgrade transformers or delete Mistral tokenizer files."
)
# Some models have an incorrect tokenizer_class on the hub.
# For these model types, bypass AutoTokenizer and use TokenizersBackend directly.
model_type = getattr(config, "model_type", None) if config else None
+12 -15
View File
@@ -104,21 +104,18 @@ def detect_token_suffix(tokenizer: TokenizerLike) -> str:
round-trips.
"""
import transformers
if int(transformers.__version__.split(".")[0]) >= 5:
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
think_begin_as_special = init_kwargs.get(
"model_specific_special_tokens", {}
).get("think_begin_token", "")
if think_begin_as_special:
raise RuntimeError(
"This checkpoint declares HYV4 structural tokens (think_begin_token"
"/toolcalls_begin_token/argkey_begin_token) in "
"tokenizer_config.json, which transformers 5 no longer supports. "
"Remove those fields and keep the tokens in the tokenizer's own "
"token definitions so the suffix can be read from the vocab."
)
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
think_begin_as_special = init_kwargs.get("model_specific_special_tokens", {}).get(
"think_begin_token", ""
)
if think_begin_as_special:
raise RuntimeError(
"This checkpoint declares HYV4 structural tokens (think_begin_token"
"/toolcalls_begin_token/argkey_begin_token) in "
"tokenizer_config.json, which transformers 5 no longer supports. "
"Remove those fields and keep the tokens in the tokenizer's own "
"token definitions so the suffix can be read from the vocab."
)
structural_token_re = re.compile(
r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>"
-8
View File
@@ -7,7 +7,6 @@ from collections.abc import Callable, Iterator, Sequence
from contextlib import contextmanager
from dataclasses import asdict
from functools import cache, partial, wraps
from importlib.metadata import version
from pathlib import Path
from typing import Any, Literal, TypeAlias
@@ -15,7 +14,6 @@ import huggingface_hub
import torch
import transformers.configuration_utils as hf_configuration_utils
from huggingface_hub import constants
from packaging.version import Version
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
from transformers import GenerationConfig, PreTrainedConfig
from transformers.configuration_utils import ALLOWED_LAYER_TYPES
@@ -69,12 +67,6 @@ _DENSE_MODULE_TYPES = {
"pylate.models.Dense.Dense",
}
if Version(version("transformers")) < Version("5.0.0"):
raise ImportError(
"Support for Transformers v4 is deprecated and was removed in vLLM v0.24.0. "
"Please upgrade to Transformers v5: pip install --upgrade transformers"
)
class LazyConfigDict(dict):
def __getitem__(self, key):
+2 -12
View File
@@ -61,10 +61,8 @@ class MistralCommonFeatureExtractor(SequenceFeatureExtractor):
"""HF-compatible duck-typed ``fetch_audio``.
Mirrors :meth:`transformers.SequenceFeatureExtractor.fetch_audio` so
:class:`transformers.ProcessorMixin.prepare_inputs_layout` (added in
transformers 5.10) works on this duck-typed feature extractor. Older
transformers versions never invoke this method, so the addition is a
no-op there.
:class:`transformers.ProcessorMixin.prepare_inputs_layout` works on this
duck-typed feature extractor.
Accepts the same shapes as ``SequenceFeatureExtractor.fetch_audio``:
@@ -77,11 +75,6 @@ class MistralCommonFeatureExtractor(SequenceFeatureExtractor):
``ProcessorMixin.prepare_inputs_layout`` always passes already-decoded
audio (numpy array or torch tensor), so the str / list-of-str branches
exist only to keep the contract identical to the upstream method.
The semantics of ``transformers.audio_utils.is_valid_audio`` differ
between transformers versions (5.9 only accepts ndarray/tensor; 5.10
also accepts ``list[float]``). We detect ``list[float]`` explicitly to
keep behavior identical across versions.
"""
from transformers.audio_utils import is_valid_audio
@@ -89,9 +82,6 @@ class MistralCommonFeatureExtractor(SequenceFeatureExtractor):
if is_valid_audio(audio_url_or_urls):
return audio_url_or_urls
if isinstance(audio_url_or_urls, (list, tuple)):
if audio_url_or_urls and isinstance(audio_url_or_urls[0], float):
# A single audio represented as ``list[float]``.
return audio_url_or_urls
return [
self.fetch_audio(x, sampling_rate=sampling_rate)
for x in audio_url_or_urls