From 5d20979fc90a38f450e0fb89661b271926a3061a Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Fri, 2 Oct 2026 12:17:24 +0100 Subject: [PATCH] [Misc] Remove code paths for Transformers < 5.16.1 (#59762) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .../language/generation/test_granite.py | 17 +- .../processing/test_transformers_audio.py | 37 +- .../processing/test_transformers_image.py | 158 ++++---- .../processing/transformers_backend.py | 51 --- tests/models/registry.py | 86 +---- tests/models/transformers/test_backend.py | 11 - tests/models/utils.py | 14 - tests/tokenizers_/test_registry.py | 49 --- tests/v1/e2e/spec_decode/mtp/_correctness.py | 10 - vllm/model_executor/models/mistral3.py | 13 - .../models/transformers/base.py | 3 - .../model_executor/models/transformers/moe.py | 1 - .../models/transformers/multimodal.py | 345 +----------------- vllm/reasoning/hy_v4_reasoning_parser.py | 27 +- vllm/tokenizers/registry.py | 28 -- vllm/tool_parsers/hy_v4_tool_parser.py | 27 +- vllm/transformers_utils/config.py | 8 - vllm/transformers_utils/processors/voxtral.py | 14 +- 18 files changed, 142 insertions(+), 757 deletions(-) delete mode 100644 tests/models/multimodal/processing/transformers_backend.py diff --git a/tests/models/language/generation/test_granite.py b/tests/models/language/generation/test_granite.py index 207e798fef..f07839c29f 100644 --- a/tests/models/language/generation/test_granite.py +++ b/tests/models/language/generation/test_granite.py @@ -5,16 +5,15 @@ from transformers import GraniteConfig from vllm.model_executor.models.granite import granite_layer_attn_params -from ...utils import check_logprobs_close, check_transformers_version +from ...utils import check_logprobs_close -# model -> minimum transformers version, or None if unconstrained -MODELS = { +MODELS = [ # TODO(sang): Sliding window should be tested separately. - "ibm/PowerLM-3b": None, - "ibm/PowerMoE-3b": None, - "ibm-granite/granite-swash-2b": "5.15.1", - "ibm-granite/granite-swash-3b-a600m": "5.15.1", -} + "ibm/PowerLM-3b", + "ibm/PowerMoE-3b", + "ibm-granite/granite-swash-2b", + "ibm-granite/granite-swash-3b-a600m", +] @pytest.mark.parametrize("model", MODELS) @@ -31,8 +30,6 @@ def test_models( max_tokens: int, num_logprobs: int, ) -> None: - check_transformers_version(model, min_transformers_version=MODELS[model]) - with hf_runner(model, dtype=dtype) as hf_model: hf_outputs = hf_model.generate_greedy_logprobs_limit( example_prompts, max_tokens, num_logprobs diff --git a/tests/models/multimodal/processing/test_transformers_audio.py b/tests/models/multimodal/processing/test_transformers_audio.py index 61517ea0af..1c1204d8e5 100644 --- a/tests/models/multimodal/processing/test_transformers_audio.py +++ b/tests/models/multimodal/processing/test_transformers_audio.py @@ -4,11 +4,8 @@ import numpy as np import pytest from vllm.config import ModelConfig -from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor from vllm.multimodal import MULTIMODAL_REGISTRY -from .transformers_backend import PROCESSOR_CLASSES, create_processor - AUDIO_MODEL_SETTINGS = { "ibm-granite/granite-speech-3.3-2b": { "prompt": ( @@ -56,7 +53,6 @@ AUDIO_MODEL_SETTINGS = { } -@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) @pytest.mark.parametrize( "model_id", [ @@ -78,10 +74,12 @@ AUDIO_MODEL_SETTINGS = { "zai-org/GLM-ASR-Nano-2512", ], ) -def test_audio_multimodal_processor(model_id, processor_cls): +def test_audio_multimodal_processor(model_id): settings = AUDIO_MODEL_SETTINGS[model_id] - mm_processor = create_processor(model_id, processor_cls) + mm_processor = MULTIMODAL_REGISTRY.create_processor( + ModelConfig(model=model_id, model_impl="transformers") + ) audio = np.zeros(16000, dtype=np.float32) mm_data = {"audio": (audio, 16000)} @@ -112,13 +110,14 @@ def test_audio_multimodal_processor(model_id, processor_cls): ) -@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) @pytest.mark.parametrize("separator", [" and ", ""]) -def test_audio_multiple_inputs(separator, processor_cls): +def test_audio_multiple_inputs(separator): """Multiple audios per prompt are each detected as a separate placeholder and multi-modal item by the Transformers modelling backend.""" model_id = "ibm-granite/granite-speech-3.3-2b" - mm_processor = create_processor(model_id, processor_cls) + mm_processor = MULTIMODAL_REGISTRY.create_processor( + ModelConfig(model=model_id, model_impl="transformers") + ) audio_token = mm_processor.info.get_hf_processor().audio_token # One token per audio; the processor expands each to its placeholder run. @@ -128,21 +127,11 @@ def test_audio_multiple_inputs(separator, processor_cls): ) audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)] - def process(): - return mm_processor( - prompt=prompt, - mm_items=mm_processor.info.parse_mm_data({"audio": audios}), - hf_processor_mm_kwargs={}, - ) - - # The legacy path reads placeholders off contiguous runs of the audio token, so - # it cannot tell adjacent ones apart and says so instead of merging them - if processor_cls is LegacyMultiModalProcessor and not separator: - with pytest.raises(ValueError, match="Separate them in the prompt"): - process() - return - - result = process() + result = mm_processor( + prompt=prompt, + mm_items=mm_processor.info.parse_mm_data({"audio": audios}), + hf_processor_mm_kwargs={}, + ) assert len(result["mm_placeholders"]["audio"]) == 2 assert len(result["mm_kwargs"]["audio"]) == 2 diff --git a/tests/models/multimodal/processing/test_transformers_image.py b/tests/models/multimodal/processing/test_transformers_image.py index 8f5a47be7b..26a3a592ca 100644 --- a/tests/models/multimodal/processing/test_transformers_image.py +++ b/tests/models/multimodal/processing/test_transformers_image.py @@ -5,23 +5,16 @@ from unittest.mock import patch import pytest from vllm.assets.image import ImageAsset -from vllm.model_executor.models.transformers.multimodal import ( - LegacyMultiModalProcessor, - OffsetsMultiModalProcessor, -) - -from .transformers_backend import ( - PROCESSOR_CLASSES, - create_cached_processor, - create_processor, - offsets_only, -) +from vllm.config import ModelConfig +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.cache import MultiModalProcessorOnlyCache -@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) @pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]) -def test_multimodal_processor(model_id, processor_cls): - mm_processor = create_processor(model_id, processor_cls) +def test_multimodal_processor(model_id): + mm_processor = MULTIMODAL_REGISTRY.create_processor( + ModelConfig(model=model_id, model_impl="transformers") + ) image_pil = ImageAsset("cherry_blossom").pil_image mm_data = {"image": image_pil} @@ -63,9 +56,11 @@ def test_multimodal_processor(model_id, processor_cls): ) -def _process_two_images(processor_cls, separator: str): +def _process_two_images(separator: str): model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" - mm_processor = create_processor(model_id, processor_cls) + mm_processor = MULTIMODAL_REGISTRY.create_processor( + ModelConfig(model=model_id, model_impl="transformers") + ) image = ImageAsset("cherry_blossom").pil_image prompt = ( @@ -80,30 +75,29 @@ def _process_two_images(processor_cls, separator: str): ) -@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) -def test_image_multiple_inputs(processor_cls): +def test_image_multiple_inputs(): """Multiple images per prompt are each detected as a separate placeholder and multi-modal item by the Transformers modelling backend.""" - result = _process_two_images(processor_cls, separator="\n and ") + result = _process_two_images(separator="\n and ") assert len(result["mm_placeholders"]["image"]) == 2 assert len(result["mm_kwargs"]["image"]) == 2 -@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) -def test_image_adjacent_inputs(processor_cls): +def test_image_adjacent_inputs(): """Adjacent images stay separate placeholders rather than merging into one.""" - result = _process_two_images(processor_cls, separator="") + result = _process_two_images(separator="") assert len(result["mm_placeholders"]["image"]) == 2 assert len(result["mm_kwargs"]["image"]) == 2 -@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) -def test_batch_padding_removed_from_image_items(processor_cls): +def test_batch_padding_removed_from_image_items(): """Emu3 pads every image up to the largest in the batch, which would leave an item's data dependent on what it was processed with and so uncacheable.""" - mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls) + mm_processor = MULTIMODAL_REGISTRY.create_processor( + ModelConfig(model="BAAI/Emu3-Chat-hf", model_impl="transformers") + ) image_token = mm_processor.info.get_hf_processor().image_token images = [ @@ -128,53 +122,39 @@ def test_batch_padding_removed_from_image_items(processor_cls): assert len(shapes) == 2 -def _process_one_gemma3_image(processor_cls): - mm_processor = create_processor("google/gemma-3-4b-it", processor_cls) - hf_processor = mm_processor.info.get_hf_processor() - result = mm_processor( - prompt=f"{hf_processor.boi_token} What is this?", +def _process_one_gemma3_image(): + mm_processor = MULTIMODAL_REGISTRY.create_processor( + ModelConfig(model="google/gemma-3-4b-it", model_impl="transformers") + ) + boi_token = mm_processor.info.get_hf_processor().boi_token + return mm_processor( + prompt=f"{boi_token} What is this?", mm_items=mm_processor.info.parse_mm_data( {"image": ImageAsset("cherry_blossom").pil_image} ), hf_processor_mm_kwargs={}, ) - return hf_processor, result -@offsets_only def test_non_embedding_tokens_excluded_from_placeholders(): """Gemma3 wraps each image in text that carries no embeddings, which must be inside the placeholder range but masked out of it.""" - _, result = _process_one_gemma3_image(OffsetsMultiModalProcessor) + result = _process_one_gemma3_image() (placeholder,) = result["mm_placeholders"]["image"] assert placeholder.is_embed is not None assert 0 < int(placeholder.is_embed.sum()) < placeholder.length -def test_legacy_placeholders_hold_only_image_tokens(): - """The legacy path spans whatever `mm_token_type_ids` attributes to the image, - which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets - path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the - marker in the unexpanded prompt, not the token the expansion repeats.""" - hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor) - - (placeholder,) = result["mm_placeholders"]["image"] - assert placeholder.length == hf_processor.image_seq_length - prompt_ids = result["prompt_token_ids"] - covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length] - assert set(covered) == {hf_processor.tokenizer.image_token_id} - - -@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) -def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls): +def test_tokens_structuring_an_image_are_masked_not_dropped(): """SmolVLM splits each image into tiles introduced by tokens carrying no embeddings. Those belong inside the placeholder and masked out, because the token count the processor reports is over the whole span. Idefics3 also refuses a prompt - holding `` when no images are passed, which is how the offsets path has to - tokenize it before splicing in the expansion.""" - mm_processor = create_processor( - "HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls + holding `` when no images are passed, which is how the prompt has to be + tokenized before splicing in the expansion.""" + model_id = "HuggingFaceTB/SmolVLM-256M-Instruct" + mm_processor = MULTIMODAL_REGISTRY.create_processor( + ModelConfig(model=model_id, model_impl="transformers") ) result = mm_processor( prompt="What is this?", @@ -189,12 +169,13 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls): assert 0 < int(placeholder.is_embed.sum()) < placeholder.length -@offsets_only def test_missing_replacement_offsets_names_the_processor(): """A processor that reports no replacement offsets cannot be served, which must be said plainly rather than surfacing later as a field config mismatch.""" model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" - mm_processor = create_processor(model_id, OffsetsMultiModalProcessor) + mm_processor = MULTIMODAL_REGISTRY.create_processor( + ModelConfig(model=model_id, model_impl="transformers") + ) hf_processor_cls = type(mm_processor.info.get_hf_processor()) hf_call = hf_processor_cls.__call__ @@ -216,11 +197,12 @@ def test_missing_replacement_offsets_names_the_processor(): ) -@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) -def test_text_only_prompt(processor_cls): +def test_text_only_prompt(): """An image model still accepts a prompt with no images.""" model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" - mm_processor = create_processor(model_id, processor_cls) + mm_processor = MULTIMODAL_REGISTRY.create_processor( + ModelConfig(model=model_id, model_impl="transformers") + ) result = mm_processor( prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n", @@ -232,12 +214,14 @@ def test_text_only_prompt(processor_cls): assert not result["mm_placeholders"] -@offsets_only def test_repeated_image_hits_the_processor_cache(): """Check that mm caching is actually working.""" - mm_processor, cache = create_cached_processor( - "llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor + model_config = ModelConfig( + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", model_impl="transformers" ) + model_config.multimodal_config.mm_processor_cache_gb = 4 + mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + cache = MultiModalProcessorOnlyCache(model_config) image = ImageAsset("cherry_blossom").pil_image def process(): @@ -255,7 +239,6 @@ def test_repeated_image_hits_the_processor_cache(): assert first["mm_hashes"] == second["mm_hashes"] -@offsets_only @pytest.mark.parametrize( ("model_id", "prompt"), [ @@ -267,7 +250,7 @@ def test_repeated_image_hits_the_processor_cache(): " and more text", marks=pytest.mark.xfail( reason="Emu3Processor prepends its BOS token only when images are " - "passed, so the unexpanded prompt the offsets path tokenizes never " + "passed, so the unexpanded prompt vLLM tokenizes never " "gets one. Fixed by huggingface/transformers#47924, unreleased.", strict=False, ), @@ -275,33 +258,40 @@ def test_repeated_image_hits_the_processor_cache(): ], ) def test_spliced_prompt_matches_hf_expansion(model_id, prompt): - """The offsets path splices the expansion into a prompt tokenized without any - multi-modal data, so its token ids have to come out the same as the ones the HF - processor produces itself, which is what the legacy path returns.""" - prompt_ids = [] - for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor): - mm_processor = create_processor(model_id, processor_cls) - prompt_ids.append( - mm_processor( - prompt=prompt, - mm_items=mm_processor.info.parse_mm_data( - {"image": ImageAsset("cherry_blossom").pil_image} - ), - hf_processor_mm_kwargs={}, - )["prompt_token_ids"] - ) + """The prompt is tokenized without any multi-modal data and the expansion spliced + in, so its token ids have to come out the same as the ones the HF processor + produces itself.""" + mm_processor = MULTIMODAL_REGISTRY.create_processor( + ModelConfig(model=model_id, model_impl="transformers") + ) + info = mm_processor.info + image = ImageAsset("cherry_blossom").pil_image - legacy_ids, offsets_ids = prompt_ids - assert legacy_ids == offsets_ids + hf_processor = info.get_hf_processor() + prompt_ids = info.get_tokenizer().encode( + prompt, **info.default_tok_params.get_encode_kwargs() + ) + hf_ids = info.ctx.call_hf_processor( + hf_processor, + dict(text=hf_processor.decode(prompt_ids), images=[image]), + dict(truncation=False, add_special_tokens=False), + )["input_ids"][0].tolist() + + result = mm_processor( + prompt=prompt, + mm_items=info.parse_mm_data({"image": image}), + hf_processor_mm_kwargs={}, + ) + assert result["prompt_token_ids"] == hf_ids -@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) -def test_nested_image_fields_split_per_image(processor_cls): +def test_nested_image_fields_split_per_image(): """Idefics3 returns image fields with a leading batch dimension, putting the rows belonging to each image one dimension further in. Slicing the batch dimension instead handed the first image every row and the second an empty tensor.""" - mm_processor = create_processor( - "HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls + model_id = "HuggingFaceTB/SmolVLM-256M-Instruct" + mm_processor = MULTIMODAL_REGISTRY.create_processor( + ModelConfig(model=model_id, model_impl="transformers") ) image = ImageAsset("cherry_blossom").pil_image result = mm_processor( diff --git a/tests/models/multimodal/processing/transformers_backend.py b/tests/models/multimodal/processing/transformers_backend.py deleted file mode 100644 index 869afe9456..0000000000 --- a/tests/models/multimodal/processing/transformers_backend.py +++ /dev/null @@ -1,51 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Helpers for testing both processors of the Transformers modelling backend.""" - -import pytest - -from vllm.config import ModelConfig -from vllm.model_executor.models.transformers.multimodal import ( - LegacyMultiModalProcessor, - MultiModalDummyInputsBuilder, - MultiModalProcessingInfo, - MultiModalProcessor, - OffsetsMultiModalProcessor, -) -from vllm.multimodal.cache import MultiModalProcessorOnlyCache -from vllm.multimodal.processing import InputProcessingContext -from vllm.tokenizers.registry import cached_tokenizer_from_config - -offsets_only = pytest.mark.skipif( - MultiModalProcessor is not OffsetsMultiModalProcessor, - reason="Replacement offsets are only used from transformers 5.15.0 onwards", -) - -PROCESSOR_CLASSES = [ - pytest.param(LegacyMultiModalProcessor, id="legacy"), - pytest.param(OffsetsMultiModalProcessor, id="offsets", marks=offsets_only), -] - - -def create_processor(model_id: str, processor_cls): - """Build a processor directly, because the registry only ever builds the one the - installed transformers version selects, leaving the other path untested.""" - model_config = ModelConfig(model=model_id, model_impl="transformers") - ctx = InputProcessingContext( - model_config, cached_tokenizer_from_config(model_config) - ) - info = MultiModalProcessingInfo(ctx) - return processor_cls(info, MultiModalDummyInputsBuilder(info)) - - -def create_cached_processor(model_id: str, processor_cls): - """Build a processor backed by a real multi-modal processor cache, and hand the - cache back so a test can check it was actually used.""" - model_config = ModelConfig(model=model_id, model_impl="transformers") - model_config.multimodal_config.mm_processor_cache_gb = 4 - ctx = InputProcessingContext( - model_config, cached_tokenizer_from_config(model_config) - ) - info = MultiModalProcessingInfo(ctx) - cache = MultiModalProcessorOnlyCache(model_config) - return processor_cls(info, MultiModalDummyInputsBuilder(info)), cache diff --git a/tests/models/registry.py b/tests/models/registry.py index 45d1a0a4f5..890869d2f2 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -233,9 +233,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "CohereLabs/North-Mini-Code", trust_remote_code=True, is_available_online=False, - min_transformers_version="5.9.0", ), - "CwmForCausalLM": _HfExamplesInfo("facebook/cwm", min_transformers_version="4.58"), + "CwmForCausalLM": _HfExamplesInfo("facebook/cwm"), # FIXME: databricks/dbrx-instruct has been deleted "DbrxForCausalLM": _HfExamplesInfo( "databricks/dbrx-instruct", is_available_online=False @@ -266,9 +265,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True ), "Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"), - "ExaoneMoeForCausalLM": _HfExamplesInfo( - "LGAI-EXAONE/K-EXAONE-236B-A23B", min_transformers_version="5.1.0" - ), + "ExaoneMoeForCausalLM": _HfExamplesInfo("LGAI-EXAONE/K-EXAONE-236B-A23B"), "FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"), "FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"), "FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"), @@ -279,7 +276,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"), "Gemma4ForCausalLM": _HfExamplesInfo( "google/gemma-4-E2B-it", - min_transformers_version="5.0.0", ), "Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"), "GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"), @@ -287,11 +283,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"), "Glm4MoeLiteForCausalLM": _HfExamplesInfo( "zai-org/GLM-4.7-Flash", - min_transformers_version="5.0.0", - ), - "GlmMoeDsaForCausalLM": _HfExamplesInfo( - "zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False ), + "GlmMoeDsaForCausalLM": _HfExamplesInfo("zai-org/GLM-5", is_available_online=False), "Glm5NextForCausalLM": _HfExamplesInfo("zai-org/GLM-5.3-Flash"), "GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2"), "GPTBigCodeForCausalLM": _HfExamplesInfo( @@ -316,15 +309,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "GraniteMoeSharedForCausalLM": _HfExamplesInfo( "ibm-research/moe-7b-1b-active-shared-experts" ), - "GraniteMoeSWAForCausalLM": _HfExamplesInfo( - "ibm-granite/granite-swash-3b-a600m", min_transformers_version="5.15.1" - ), - "GraniteSWAForCausalLM": _HfExamplesInfo( - "ibm-granite/granite-swash-2b", min_transformers_version="5.15.1" - ), + "GraniteMoeSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-3b-a600m"), + "GraniteSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-2b"), "HrmTextForCausalLM": _HfExamplesInfo( "sapientinc/HRM-Text-1B", - min_transformers_version="5.9.0", ), "HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"), "HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct"), @@ -333,7 +321,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "HYV4ForCausalLM": _HfExamplesInfo("tencent/Hy4-preview", trust_remote_code=True), "HyperCLOVAXForCausalLM": _HfExamplesInfo( "naver-hyperclovax/HyperCLOVAX-SEED-Think-14B", - min_transformers_version="5.9.0", ), "InklingForCausalLM": _HfExamplesInfo( "thinkingmachines/Inkling-NVFP4", @@ -353,9 +340,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "IQuestLoopCoderForCausalLM": _HfExamplesInfo( "IQuestLab/IQuest-Coder-V1-40B-Loop-Instruct", trust_remote_code=True ), - "Jais2ForCausalLM": _HfExamplesInfo( - "inceptionai/Jais-2-8B-Chat", min_transformers_version="4.58" - ), + "Jais2ForCausalLM": _HfExamplesInfo("inceptionai/Jais-2-8B-Chat"), "JambaForCausalLM": _HfExamplesInfo( "ai21labs/AI21-Jamba-1.5-Mini", extras={ @@ -375,7 +360,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"), "Lfm2MoeForCausalLM": _HfExamplesInfo( "LiquidAI/LFM2-8B-A1B", - min_transformers_version="5.0.0", use_original_num_layers=True, # Initialize at least one MoE layer hf_overrides={"num_hidden_layers": 4}, @@ -422,7 +406,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { ), "MiniCPM4ForCausalLM": _HfExamplesInfo( "openbmb/MiniCPM4.1-8B", - min_transformers_version="4.56", max_transformers_version="4.57", transformers_version_reason={ "hf": "HF remote code imports removed `is_torch_fx_available`; " @@ -523,7 +506,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "Qwen3NextForCausalLM": _HfExamplesInfo( "Qwen/Qwen3-Next-80B-A3B-Instruct", extras={"tiny-random": "tiny-random/qwen3-next-moe"}, - min_transformers_version="4.56.3", ), "Rnj1ForCausalLM": _HfExamplesInfo( "EssentialAI/rnj-1-instruct", @@ -722,7 +704,6 @@ _TOKEN_CLASSIFICATION_EXAMPLE_MODELS = { ), "OpenAIPrivacyFilterForTokenClassification": _HfExamplesInfo( "openai/privacy-filter", - min_transformers_version="5.6.0.dev0", ), "RobertaForTokenClassification": _HfExamplesInfo( "Jean-Baptiste/roberta-large-ner-english" @@ -794,7 +775,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"), "AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo( "nvidia/audio-flamingo-3-hf", - min_transformers_version="5.3.0", transformers_version_reason={ "vllm": "Needs https://github.com/huggingface/transformers/pull/43538" }, @@ -823,22 +803,18 @@ _MULTIMODAL_EXAMPLE_MODELS = { "Cohere2VisionForConditionalGeneration": _HfExamplesInfo( "CohereLabs/command-a-vision-07-2025", extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"}, - min_transformers_version="5.9.0", ), "CohereCompassForConditionalGeneration": _HfExamplesInfo( "CohereLabs/North-Micro-Vision-Instruct", - min_transformers_version="5.16.0", ), "Cosmos3ForConditionalGeneration": _HfExamplesInfo( "nvidia/Cosmos3-Nano", extras={"super": "nvidia/Cosmos3-Super"}, max_model_len=4096, - min_transformers_version="4.57", ), "Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo( "nvidia/Cosmos3-Edge", max_model_len=4096, - min_transformers_version="5.15", use_original_num_layers=True, hf_overrides={ "text_config": { @@ -887,7 +863,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { ), "Exaone4_5_ForConditionalGeneration": _HfExamplesInfo( "LGAI-EXAONE/EXAONE-4.5-33B", - min_transformers_version="5.6.0", ), "FireRedASR2ForConditionalGeneration": _HfExamplesInfo( "allendou/FireRedASR2-LLM-vllm", @@ -918,19 +893,14 @@ _MULTIMODAL_EXAMPLE_MODELS = { ), "Gemma4ForConditionalGeneration": _HfExamplesInfo( "google/gemma-4-E2B-it", - min_transformers_version="5.5.0", ), - # TODO: update min_transformers_version when Gemma4 Unified lands in - # a stable transformers release. "Gemma4UnifiedForConditionalGeneration": _HfExamplesInfo( "google/gemma-4-12B-it", - min_transformers_version="5.8.0", is_available_online=False, ), "Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"), "GlmAsrForConditionalGeneration": _HfExamplesInfo( "zai-org/GLM-ASR-Nano-2512", - min_transformers_version="5.0.0", ), "Granite4VisionForConditionalGeneration": _HfExamplesInfo( "ibm-granite/granite-vision-4.1-4b", @@ -943,7 +913,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { ), "GraniteSpeechPlusForConditionalGeneration": _HfExamplesInfo( "ibm-granite/granite-speech-4.1-2b-plus", - min_transformers_version="5.8.0", ), "GLM4VForCausalLM": _HfExamplesInfo( "zai-org/glm-4v-9b", @@ -957,7 +926,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { "Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"), "GlmOcrForConditionalGeneration": _HfExamplesInfo( "zai-org/GLM-OCR", - min_transformers_version="5.1.0", ), "Glm5NextForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-5.3-Flash"), "H2OVLChatModel": _HfExamplesInfo( @@ -974,7 +942,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { ), "HunYuanVLForConditionalGeneration": _HfExamplesInfo( "tencent/HunyuanOCR", - min_transformers_version="5.13", ), "Idefics3ForConditionalGeneration": _HfExamplesInfo( "HuggingFaceM4/Idefics3-8B-Llama3", @@ -1087,7 +1054,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { ), "Lfm2VlForConditionalGeneration": _HfExamplesInfo( "LiquidAI/LFM2-VL-450M", - min_transformers_version="5.0.0", ), "Llama4ForConditionalGeneration": _HfExamplesInfo( "meta-llama/Llama-4-Scout-17B-16E-Instruct", @@ -1154,7 +1120,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { ), "MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo( "openbmb/MiniCPM-V-4_6", - min_transformers_version="5.7.0", ), "MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo( "MiniMaxAI/MiniMax-M3", @@ -1201,7 +1166,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { "Molmo2ForConditionalGeneration": _HfExamplesInfo( "allenai/Molmo2-8B", extras={"olmo": "allenai/Molmo2-O-7B"}, - min_transformers_version="4.51", trust_remote_code=True, # required by current PrefixLM implementation max_num_batched_tokens=31872, @@ -1353,7 +1317,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { ), "QianfanOCRForConditionalGeneration": _HfExamplesInfo( "baidu/Qianfan-OCR", - min_transformers_version="5.6.0", ), "Qwen2AudioForConditionalGeneration": _HfExamplesInfo( "Qwen/Qwen2-Audio-7B-Instruct" @@ -1368,12 +1331,10 @@ _MULTIMODAL_EXAMPLE_MODELS = { "Qwen3VLForConditionalGeneration": _HfExamplesInfo( "Qwen/Qwen3-VL-4B-Instruct", max_model_len=4096, - min_transformers_version="4.57", ), "Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo( "Qwen/Qwen3-VL-30B-A3B-Instruct", max_model_len=4096, - min_transformers_version="4.57", ), "Qwen3_5ForConditionalGeneration": _HfExamplesInfo( "Qwen/Qwen3.5-0.8B", @@ -1391,23 +1352,19 @@ _MULTIMODAL_EXAMPLE_MODELS = { "Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo( "Qwen/Qwen3-Omni-30B-A3B-Instruct", max_model_len=4096, - min_transformers_version="4.57", ), "Qwen3ASRForConditionalGeneration": _HfExamplesInfo( "Qwen/Qwen3-ASR-0.6B", max_model_len=4096, - min_transformers_version="4.57", ), "Qwen3ASRRealtimeGeneration": _HfExamplesInfo( "Qwen/Qwen3-ASR-0.6B", max_model_len=4096, - min_transformers_version="4.57", hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]}, ), "Qwen3ASRForcedAlignerForTokenClassification": _HfExamplesInfo( "Qwen/Qwen3-ForcedAligner-0.6B", max_model_len=4096, - min_transformers_version="4.57", hf_overrides={"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]}, ), "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True), @@ -1456,7 +1413,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { ), "VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo( "microsoft/VibeVoice-ASR-HF", - min_transformers_version="5.13.0", ), "VoxtralForConditionalGeneration": _HfExamplesInfo( "mistralai/Voxtral-Mini-3B-2507", @@ -1530,7 +1486,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { use_original_num_layers=True, # DFlash requires all layers max_model_len=8192, # Reduce for CI max_num_seqs=32, - min_transformers_version="4.56.3", # Required for Qwen3Next ), "MuseGlimmerAssistantModel": _HfExamplesInfo( "meta-models/Muse-Glimmer-30B", @@ -1576,7 +1531,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { speculative_model="Qwen/Qwen3-Omni-30B-A3B-DSpark", is_available_online=False, use_original_num_layers=True, - min_transformers_version="4.57", ), "Gemma4DSparkModel": _HfExamplesInfo( "google/gemma-4-12B-it", @@ -1737,7 +1691,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { "google/gemma-4-E4B-it", speculative_model="google/gemma-4-E4B-it-assistant", trust_remote_code=True, - min_transformers_version="5.8.0", ), "ErnieMTPModel": _HfExamplesInfo( "baidu/ERNIE-4.5-21B-A3B-PT", @@ -1747,13 +1700,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { "ExaoneMoeMTP": _HfExamplesInfo( "LGAI-EXAONE/K-EXAONE-236B-A23B", speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B", - min_transformers_version="5.1.0", enable_prefix_caching=False, ), "Exaone4_5_MTP": _HfExamplesInfo( "LGAI-EXAONE/EXAONE-4.5-33B", speculative_model="LGAI-EXAONE/EXAONE-4.5-33B", - min_transformers_version="5.6.0", ), "ExtractHiddenStatesModel": _HfExamplesInfo( "Qwen/Qwen3-8B", @@ -1766,12 +1717,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { "Glm4MoeLiteMTPModel": _HfExamplesInfo( "zai-org/GLM-4.7-Flash", speculative_model="zai-org/GLM-4.7-Flash", - min_transformers_version="5.0.0", ), "GlmOcrMTPModel": _HfExamplesInfo( "zai-org/GLM-OCR", speculative_model="zai-org/GLM-OCR", - min_transformers_version="5.1.0", ), "Glm5NextMTPModel": _HfExamplesInfo( "zai-org/GLM-5.3-Flash", @@ -1842,9 +1791,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { trust_remote_code=True, is_available_online=False, ), - "Qwen3NextMTP": _HfExamplesInfo( - "Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3" - ), + "Qwen3NextMTP": _HfExamplesInfo("Qwen/Qwen3-Next-80B-A3B-Instruct"), "Qwen3_5MTP": _HfExamplesInfo( "Qwen/Qwen3.5-0.8B", speculative_model="Qwen/Qwen3.5-0.8B", @@ -1865,29 +1812,20 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { } _TRANSFORMERS_BACKEND_MODELS = { - "TransformersEmbeddingModel": _HfExamplesInfo( - "BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0" - ), + "TransformersEmbeddingModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"), "TransformersForSequenceClassification": _HfExamplesInfo( "papluca/xlm-roberta-base-language-detection", - min_transformers_version="5.0.0", ), "TransformersForCausalLM": _HfExamplesInfo( "hmellor/Ilama-3.2-1B", trust_remote_code=True ), "TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"), - "TransformersMoEForCausalLM": _HfExamplesInfo( - "allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0" - ), + "TransformersMoEForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924"), "TransformersMultiModalMoEForCausalLM": _HfExamplesInfo( - "Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0" - ), - "TransformersMoEEmbeddingModel": _HfExamplesInfo( - "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0" - ), - "TransformersMoEForSequenceClassification": _HfExamplesInfo( - "Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0" + "Qwen/Qwen3-VL-30B-A3B-Instruct" ), + "TransformersMoEEmbeddingModel": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"), + "TransformersMoEForSequenceClassification": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"), "TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"), "TransformersMultiModalForSequenceClassification": _HfExamplesInfo( "google/gemma-3-4b-it" diff --git a/tests/models/transformers/test_backend.py b/tests/models/transformers/test_backend.py index 2064f01e0c..001ee5ef12 100644 --- a/tests/models/transformers/test_backend.py +++ b/tests/models/transformers/test_backend.py @@ -185,17 +185,6 @@ def test_sinks(hf_runner: type[HfRunner], vllm_runner: type[VllmRunner]) -> None def test_mla(vllm_runner: type[VllmRunner], example_prompts: list[str]) -> None: - import transformers - from packaging.version import Version - - installed = Version(transformers.__version__) - required = Version("5.15.0.dev0") - if installed < required: - pytest.skip( - "MLA models with the Transformers modeling backend require " - f"transformers>={required}, but got {installed}" - ) - model = "hmellor/tiny-random-DeepseekV2ForCausalLM" args = (example_prompts, 32, 5) kwargs: dict[str, Any] = {"max_model_len": 2048, "enforce_eager": True} diff --git a/tests/models/utils.py b/tests/models/utils.py index 003ef1a078..4e98a981d4 100644 --- a/tests/models/utils.py +++ b/tests/models/utils.py @@ -609,17 +609,3 @@ def dummy_hf_overrides( ) return hf_config - - -def check_transformers_version( - model: str, - min_transformers_version: str | None = None, - max_transformers_version: str | None = None, -): - from .registry import _HfExamplesInfo - - return _HfExamplesInfo( - model, - min_transformers_version=min_transformers_version, - max_transformers_version=max_transformers_version, - ).check_transformers_version(on_fail="skip") diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index a4ee4a755e..312b0c03a9 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -10,7 +10,6 @@ from transformers import AutoConfig from transformers.models.auto.configuration_auto import CONFIG_MAPPING from vllm.tokenizers import TokenizerLike -from vllm.tokenizers.hf import CachedHfTokenizer from vllm.tokenizers.registry import ( TokenizerRegistry, cached_get_tokenizer, @@ -93,54 +92,6 @@ def test_resolve_tokenizer_args_rejects_hf_mistral_format_true(): ) -@pytest.mark.parametrize( - ("transformers_version", "model_type", "native_filename", "expect_error"), - [ - pytest.param("5.14.0", "mistral", "tekken.json", True), - pytest.param("5.15.0", "mistral", "tekken.json", False), - pytest.param("5.14.0", "mistral3", "tekken.json", True), - pytest.param("5.15.0", "mistral3", "tekken.json", False), - pytest.param("5.14.0", "mixtral", "tokenizer.model.v1", True), - pytest.param("5.15.0", "mixtral", "tokenizer.model.v1", False), - pytest.param("5.14.0", "mistral", None, False), - pytest.param("5.14.0", "qwen3_5_moe", "tekken.json", False), - ], -) -def test_get_tokenizer_handles_hf_mistral_transformers_compatibility( - tmp_path: Path, - transformers_version: str, - model_type: str, - native_filename: str | None, - expect_error: bool, -): - if native_filename is not None: - (tmp_path / native_filename).touch() - - with ( - patch( - "vllm.tokenizers.registry.get_config", - return_value=SimpleNamespace(model_type=model_type), - ), - patch( - "vllm.tokenizers.registry.transformers.__version__", - transformers_version, - ), - patch.object( - CachedHfTokenizer, - "from_pretrained", - return_value=SimpleNamespace(is_fast=True), - ) as from_pretrained, - ): - if expect_error: - with pytest.raises(ValueError, match="requires transformers>=5.15.0"): - get_tokenizer(str(tmp_path), tokenizer_mode="hf") - else: - tokenizer = get_tokenizer(str(tmp_path), tokenizer_mode="hf") - assert tokenizer.is_fast is True - - assert from_pretrained.called == (not expect_error) - - def test_customized_tokenizer(): TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__) diff --git a/tests/v1/e2e/spec_decode/mtp/_correctness.py b/tests/v1/e2e/spec_decode/mtp/_correctness.py index b88c1cce7a..352aa556e0 100644 --- a/tests/v1/e2e/spec_decode/mtp/_correctness.py +++ b/tests/v1/e2e/spec_decode/mtp/_correctness.py @@ -48,16 +48,6 @@ def check_mtp_correctness( elif "gemma-4" in model_name: extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0} - if draft_model is not None and "gemma-4" in draft_model: - import transformers - from packaging.version import Version - - if Version(transformers.__version__) < Version("5.8.0"): - pytest.skip( - "Gemma4 MTP assistant requires transformers>=5.8.0, " - f"got {transformers.__version__}" - ) - with vllm_runner( model_name, block_size=None, diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index 40af706e3a..f5ffe32aad 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -7,8 +7,6 @@ from typing import Annotated, Any, Literal, cast import torch import torch.nn as nn -import transformers -from packaging.version import Version from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig from transformers.models.pixtral import PixtralProcessor @@ -59,10 +57,6 @@ from .utils import ( maybe_prefix, ) -TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY = Version(transformers.__version__) >= Version( - "5.15.0" -) - class Mistral3ImagePixelInputs(TensorSchema): """Dimensions: @@ -284,13 +278,6 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder): class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]): - def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None: - # PixtralProcessor supports image-only calls starting in transformers 5.15. - # Older releases need dummy text and take the slower tokenization path. - if not TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY: - return self.dummy_inputs.get_dummy_text(mm_counts) - return None - def _postprocess_hf_mm_data( self, mm_data: Mapping[str, object], diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py index 239ef821e7..7d5fdc4ca5 100644 --- a/vllm/model_executor/models/transformers/base.py +++ b/vllm/model_executor/models/transformers/base.py @@ -691,10 +691,8 @@ class Base( # vLLM does not support encoder-decoder models, so if any encoder layer is # found in a text only model, we assume the whole model is an encoder model if has_encoder(self.model) and not is_multimodal(self.config): - self.check_version("5.0.0", "encoder models support") return EncoderOnlyAttention if self.model_config.use_mla: - self.check_version("5.15.0.dev0", "optimized MLA support") if any( isinstance(fuser, MLAFuser) for fusers in self.fusers.values() @@ -820,7 +818,6 @@ class Base( ) def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: - self.check_version("5.2.0", "Eagle3 support") from transformers.utils.output_capturing import ( OutputRecorder, maybe_install_capturing_hooks, diff --git a/vllm/model_executor/models/transformers/moe.py b/vllm/model_executor/models/transformers/moe.py index 9f63e73336..71f5bd066c 100644 --- a/vllm/model_executor/models/transformers/moe.py +++ b/vllm/model_executor/models/transformers/moe.py @@ -156,7 +156,6 @@ class MoEMixin(MixtureOfExperts, Base): num_local_physical_experts: int def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): - self.check_version("5.0.0", "MoE models support") # Skip MixtureOfExperts.__init__ and call the next class in MRO super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix) diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index cecb4f564a..9887619858 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -23,17 +23,11 @@ from typing import TYPE_CHECKING, Any import torch import transformers -from packaging.version import Version from transformers.utils.generic import ModelOutput from vllm.compilation.decorators import should_torch_compile_mm_encoder from vllm.config.utils import getattr_iter -from vllm.inputs import ( - MultiModalDataBuiltins, - MultiModalDataDict, - MultiModalInput, - mm_input, -) +from vllm.inputs import MultiModalDataBuiltins, MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.models.interfaces import ( MultiModalEmbeddings, @@ -42,26 +36,15 @@ from vllm.model_executor.models.interfaces import ( ) from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems -from vllm.multimodal.inputs import ( - MultiModalFeatureSpec, - MultiModalFieldConfig, - PlaceholderRange, -) -from vllm.multimodal.parse import ( - ImageProcessorItems, - ImageSize, - MultiModalDataItems, - MultiModalDataParser, -) +from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig +from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser from vllm.multimodal.processing import ( BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, - ProcessorInputs, PromptReplacement, PromptUpdate, PromptUpdateDetails, - TimingContext, cached_encode, ) from vllm.sequence import IntermediateTensors @@ -219,11 +202,13 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn return data -class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]): - """Processing common to both Transformers backend processors: calling the HF - processor, sizing images, and attributing its outputs to a modality. +class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): + """Locates placeholders from the `text_replacement_offsets` the HF processor + reports, expressing each one as a `PromptUpdate`. - Subclasses add the strategy for locating placeholders in the prompt. + Stating the expansion as an update is what lets it be rebuilt from an + unexpanded prompt, so this processor takes the base class's processing path + and with it the multi-modal processor cache. """ def _get_modality_field_names(self, modality: str) -> set[str]: @@ -415,301 +400,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo] for image, (height, width) in zip(pixel_values, image_sizes.tolist()) ] - -class LegacyMultiModalProcessor(_MultiModalProcessorBase): - """Locates placeholders by searching the prompt the HF processor has already - expanded for the tokens of each modality. - - Serves transformers versions with no `return_text_replacement_offsets`. - Placeholders found this way cannot be rebuilt from an unexpanded prompt, so - this processor overrides `apply` and gets no multi-modal processor cache. - Remove it once `requirements/common.txt` requires `transformers>=5.15.0`. - """ - - def _get_prompt_updates( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - out_mm_kwargs: MultiModalKwargsItems, - ) -> Sequence[PromptUpdate]: - """Empty, because `apply` writes the placeholder ranges itself rather than - deriving them from updates.""" - return [] - - def _get_mm_token_ids(self, modality: str) -> list[int]: - """Token ids marking where `modality` sits in the prompt, which for some - processors differ from the placeholder written into it. - - The expanded prompt is all this path has to go on, so it takes the - processor at its word about which tokens belong to the modality. - """ - info = self.info - processor = info.get_hf_processor() - declared = getattr(processor, f"{modality}_token_ids", None) or () - if ids := [token_id for token_id in declared if token_id is not None]: - return ids - config = info.get_hf_config() - names = (f"{modality}_token_id", f"{modality}_token_index") - token_id = getattr(processor, names[0], getattr_iter(config, names)) - if token_id is None: - token = getattr(processor, f"{modality}_token", None) - if token is not None: - token_id = info.get_tokenizer().get_vocab().get(token) - if token_id is None: - raise ValueError( - f"Cannot find {modality}_token_id on processor or model config" - ) - return [token_id] - - def _apply_audio( - self, - prompt_ids: list[int], - processed_data: "BatchFeature", - num_audios: int, - ) -> dict[str, list[PlaceholderRange]]: - """Take each contiguous run of the audio token as one item's placeholder, - and record how many tokens the run holds.""" - audio_token_ids = self._get_mm_token_ids("audio") - prompt_tensor = torch.tensor(prompt_ids) - is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids)) - - if not is_audio.any(): - raise ValueError( - f"{num_audios} audio item(s) were passed but the prompt " - "contains no audio token. Add one placeholder per audio item." - ) - - padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])]) - transitions = padded.int().diff() - offsets = torch.where(transitions == 1)[0] - lengths = torch.where(transitions == -1)[0] - offsets - - if len(offsets) != num_audios: - raise ValueError( - f"Found {len(offsets)} run(s) of the audio token in the prompt but " - f"{num_audios} audio item(s) were passed. The Transformers backend " - "locates audio placeholders by finding contiguous runs of the audio " - "token, so placeholders with no text between them cannot yet be told " - "apart. Separate them in the prompt to work around this." - ) - - ranges = [ - PlaceholderRange(offset=offset.item(), length=length.item()) - for offset, length in zip(offsets, lengths) - ] - processed_data["num_audio_tokens"] = lengths - return {"audio": ranges} - - def _get_num_multimodal_tokens( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> Mapping[str, list[int]]: - """Ask the HF processor how many tokens and patches each image expands to.""" - processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - images = mm_items.get_items("image", ImageProcessorItems) - image_sizes = [ - (size.height, size.width) - for size in map(images.get_image_size, range(len(images))) - ] - return processor._get_num_multimodal_tokens( - image_sizes=image_sizes, - **self.info.ctx.get_merged_mm_kwargs({}), - ) - - def _apply_vision( - self, - prompt_ids: list[int], - processed_data: "BatchFeature", - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - mm_token_type_ids: torch.Tensor | None, - ) -> dict[str, list[PlaceholderRange]]: - """Split the positions the processor marks as image into one placeholder per - item, sized by the token count it reports for each image.""" - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - if mm_token_type_ids is None: - raise ValueError( - f"{type(hf_processor).__name__} returned no `mm_token_type_ids`, so " - "the Transformers modeling backend cannot locate the placeholder of " - "each image." - ) - - # These mark the tokens structuring an image as well as the image itself, - # which is what the counts below are over - mm_positions = torch.where(mm_token_type_ids[0] == 1)[0] - mm_tokens_per_modality = self._get_num_multimodal_tokens( - mm_items, hf_processor_mm_kwargs - ) - - mm_placeholders: dict[str, list[PlaceholderRange]] = {} - split_sizes = mm_tokens_per_modality["num_image_tokens"] - if sum(split_sizes) != len(mm_positions): - raise ValueError( - f"The expanded prompt holds {len(mm_positions)} image token(s) but " - f"{type(hf_processor).__name__} accounts for {sum(split_sizes)} " - f"across {mm_items.get_count('image')} image item(s)." - ) - - if split_sizes: - image_token_ids = torch.tensor(self._get_mm_token_ids("image")) - mm_tokens = torch.tensor(prompt_ids)[mm_positions] - ranges = [ - PlaceholderRange( - offset=positions[0].item(), - length=positions.shape[0], - # Only some of the span carries embeddings - is_embed=torch.isin(tokens, image_token_ids), - ) - for positions, tokens in zip( - torch.split(mm_positions, split_sizes), - torch.split(mm_tokens, split_sizes), - ) - ] - mm_placeholders = {"image": ranges} - - processed_data["num_image_patches"] = torch.tensor( - mm_tokens_per_modality["num_image_patches"] - ) - return mm_placeholders - - def apply( - self, - inputs: ProcessorInputs, - timing_ctx: TimingContext, - ) -> MultiModalInput: - """Process the prompt and every multi-modal item in one HF processor call, - then read the placeholder ranges out of the token ids it returns.""" - prompt = inputs.prompt - mm_items = inputs.mm_data_items - hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs - - with timing_ctx.record("apply_hf_processor"): - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - prompt_text = hf_processor.decode(prompt) - - # Bypass cached processor and always apply to the full set of mm inputs - # NOTE: we can't just set caching=False because base class method - # transforms outputs to `MultiModalKwargs` which is not going to - # work for Transformers. The vision path has logic tied to - # `mm_tokens_per_modality` in _apply_vision() - hf_processor_mm_kwargs = { - # vLLM needs the untruncated sequence to keep placeholder - # tokens aligned. Note that the text inputs are just dummy - # text, not the original prompt. The original prompt is - # already tokenized by the renderer. - "truncation": False, - # HF processors only accept text, and the decoded string already - # contains any special tokens, so don't let them be added again - # (the HF processor call disables `add_special_tokens`). - "add_special_tokens": False, - **hf_processor_mm_kwargs, - } - - processor_data, _, passthrough_data = self._get_hf_mm_inputs( - mm_items, hf_processor_mm_kwargs - ) - - # The real prompt is fed to the processor below and the placeholder - # ranges are read back out of its output, so the dummy text meant for - # media-only callers is discarded here. - processor_data.pop("text", None) - has_mm_data = any(processor_data.values()) - - # Ask which modality owns each token of the expanded prompt, which - # is the only marking of the tokens an expansion adds around an item - if has_mm_data: - processor_data = {**processor_data, "return_mm_token_type_ids": True} - - try: - processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), - dict(text=prompt_text, **processor_data), - hf_processor_mm_kwargs, - ) - except ValueError: - if has_mm_data: - raise - # Some processors reject a prompt holding placeholders with - # no data to go with them, so tokenize it without them - prompt_ids = self.info.get_tokenizer().encode(prompt_text) - processed_data = transformers.BatchFeature( - dict(input_ids=[prompt_ids]), tensor_type="pt" - ) - self._unpad_images(processed_data) - self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs) - processed_data.update(passthrough_data) - - input_ids = processed_data.pop("input_ids") - if not isinstance(input_ids, list): - input_ids = input_ids.tolist() - - (prompt_ids,) = input_ids - - # Use overrides if provided; fallback to data-dependent hashing. - with timing_ctx.record("get_mm_hashes"): - mm_hashes = inputs.get_mm_hashes( - self.info.model_id, - self.info.ctx.get_mm_config().mm_hasher_algorithm, - ) - - # Gemma3 reports them under the key the model uses for its own token types - mm_token_type_ids = processed_data.pop("token_type_ids", None) - mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids) - - mm_placeholders: dict[str, list[PlaceholderRange]] = {} - if num_audios := mm_items.get_count("audio", strict=False): - mm_placeholders.update( - self._apply_audio(prompt_ids, processed_data, num_audios) - ) - if mm_items.get_count("image", strict=False): - mm_placeholders.update( - self._apply_vision( - prompt_ids, - processed_data, - mm_items, - hf_processor_mm_kwargs, - mm_token_type_ids, - ) - ) - - mm_kwargs = MultiModalKwargsItems.from_hf_inputs( - processed_data, - self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs), - ) - - # Bypassing `_maybe_apply_prompt_updates` also bypasses its validation. - # `_validate_mm_placeholders` can't be reused because it is typed for the - # `PlaceholderFeaturesInfo` the prompt update machinery produces. - mm_item_counts = mm_items.get_all_counts() - self._validate_mm_kwargs(mm_kwargs, mm_item_counts) - for modality, item_count in mm_item_counts.items(): - num_placeholders = len(mm_placeholders.get(modality, [])) - if num_placeholders != item_count: - raise RuntimeError( - f"Expected there to be {item_count} prompt placeholders " - f"corresponding to {item_count} {modality} items, but instead " - f"found {num_placeholders} prompt placeholders! Make sure the " - "prompt contains a placeholder token for each item." - ) - - return mm_input( - prompt_token_ids=prompt_ids, - mm_kwargs=mm_kwargs, - mm_hashes=mm_hashes, - mm_placeholders=mm_placeholders, - ) - - -class OffsetsMultiModalProcessor(_MultiModalProcessorBase): - """Locates placeholders from the `text_replacement_offsets` the HF processor - reports, expressing each one as a `PromptUpdate`. - - Stating the expansion as an update is what lets it be rebuilt from an - unexpanded prompt, so this processor takes the base class's processing path - and with it the multi-modal processor cache. - """ - def _get_prompt_updates( self, mm_items: MultiModalDataItems, @@ -800,9 +490,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): except (AttributeError, KeyError, TypeError): return None - def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: - return self.dummy_inputs.get_dummy_text(mm_counts) - def _apply_hf_processor_main( self, mm_items: MultiModalDataItems, @@ -856,8 +543,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): "to reach `ProcessorMixin.get_text_with_replacements` with one " "replacement per item, which usually means implementing " "`replace__token`. Please report this to transformers " - "so it can be fixed, and install `transformers<5.15.0` in the " - "meantime to locate placeholders in the expanded prompt instead." + "so it can be fixed." ) hf_inputs.pop("input_ids", None) return self._finalize_hf_mm_data( @@ -896,15 +582,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): ) -# From this version on, a processor reporting no offsets is an error rather than a -# fallback to searching the expanded prompt -MultiModalProcessor = ( - LegacyMultiModalProcessor - if Version(transformers.__version__) < Version("5.15.0") - else OffsetsMultiModalProcessor -) - - class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base): def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): # Skip SupportsMRoPE.__init__ and call the next class in MRO @@ -982,7 +659,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base): super()._decorate_for_torch_compile() # Decorate the encoder model classes to support torch compile if needed if self.compilation_config.compile_mm_encoder: - self.check_version("5.0.0", "multimodal encoder compilation support") encoder_classes = self._pre_trained_model_classes.encoders if not encoder_classes: raise ValueError( @@ -1113,7 +789,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base): if input_features is None: return None - self.check_version("5.13.0", "audio models support") num_audio_tokens = kwargs.pop("num_audio_tokens") kwargs.pop("token_type_ids", None) kwargs.pop("mm_token_type_ids", None) diff --git a/vllm/reasoning/hy_v4_reasoning_parser.py b/vllm/reasoning/hy_v4_reasoning_parser.py index 65035ed53a..eabe6c31ae 100644 --- a/vllm/reasoning/hy_v4_reasoning_parser.py +++ b/vllm/reasoning/hy_v4_reasoning_parser.py @@ -48,21 +48,18 @@ def detect_token_suffix(tokenizer: TokenizerLike) -> str: round-trips. """ - import transformers - - if int(transformers.__version__.split(".")[0]) >= 5: - init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} - think_begin_as_special = init_kwargs.get( - "model_specific_special_tokens", {} - ).get("think_begin_token", "") - if think_begin_as_special: - raise RuntimeError( - "This checkpoint declares HYV4 structural tokens (think_begin_token" - "/toolcalls_begin_token/argkey_begin_token) in " - "tokenizer_config.json, which transformers 5 no longer supports. " - "Remove those fields and keep the tokens in the tokenizer's own " - "token definitions so the suffix can be read from the vocab." - ) + init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} + think_begin_as_special = init_kwargs.get("model_specific_special_tokens", {}).get( + "think_begin_token", "" + ) + if think_begin_as_special: + raise RuntimeError( + "This checkpoint declares HYV4 structural tokens (think_begin_token" + "/toolcalls_begin_token/argkey_begin_token) in " + "tokenizer_config.json, which transformers 5 no longer supports. " + "Remove those fields and keep the tokens in the tokenizer's own " + "token definitions so the suffix can be read from the vocab." + ) structural_token_re = re.compile( r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>" diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 09a715c117..e7064fc252 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -7,12 +7,6 @@ from pathlib import Path from typing import TYPE_CHECKING import huggingface_hub -import transformers -from packaging.version import Version -from transformers.models.auto.tokenization_auto import ( - TOKENIZER_MAPPING_NAMES as _HF_TOKENIZER_MAPPING_NAMES, -) -from transformers.utils.import_utils import is_mistral_common_available from typing_extensions import TypeVar, assert_never from vllm import envs @@ -32,8 +26,6 @@ if TYPE_CHECKING: logger = init_logger(__name__) -_MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL = Version("5.15.0") - # Model types whose hub tokenizer_class is incorrect and should be overridden with # TokenizersBackend (the generic fast tokenizer). Adding a model type here is always a @@ -243,26 +235,6 @@ def get_tokenizer( token=kwargs.get("token"), ) - # TODO: delete when Transformers version dependency is bumped >= 5.15.0 - if ( - tokenizer_mode == "hf" - and Version(transformers.__version__) < _MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL - and is_mistral_common_available() - and _HF_TOKENIZER_MAPPING_NAMES.get(getattr(config, "model_type", None)) - == "MistralCommonBackend" - and any_pattern_in_repo_files( - model_name_or_path=str(tokenizer_name), - allow_patterns=["tekken.json", "tokenizer.model.v*"], - revision=revision, - token=kwargs.get("token"), - ) - ): - raise ValueError( - "Loading Mistral models with tokenizer_mode='hf' requires " - f"transformers>={_MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL}. " - "Please upgrade transformers or delete Mistral tokenizer files." - ) - # Some models have an incorrect tokenizer_class on the hub. # For these model types, bypass AutoTokenizer and use TokenizersBackend directly. model_type = getattr(config, "model_type", None) if config else None diff --git a/vllm/tool_parsers/hy_v4_tool_parser.py b/vllm/tool_parsers/hy_v4_tool_parser.py index b9c857dbd9..2ce0381a01 100644 --- a/vllm/tool_parsers/hy_v4_tool_parser.py +++ b/vllm/tool_parsers/hy_v4_tool_parser.py @@ -104,21 +104,18 @@ def detect_token_suffix(tokenizer: TokenizerLike) -> str: round-trips. """ - import transformers - - if int(transformers.__version__.split(".")[0]) >= 5: - init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} - think_begin_as_special = init_kwargs.get( - "model_specific_special_tokens", {} - ).get("think_begin_token", "") - if think_begin_as_special: - raise RuntimeError( - "This checkpoint declares HYV4 structural tokens (think_begin_token" - "/toolcalls_begin_token/argkey_begin_token) in " - "tokenizer_config.json, which transformers 5 no longer supports. " - "Remove those fields and keep the tokens in the tokenizer's own " - "token definitions so the suffix can be read from the vocab." - ) + init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} + think_begin_as_special = init_kwargs.get("model_specific_special_tokens", {}).get( + "think_begin_token", "" + ) + if think_begin_as_special: + raise RuntimeError( + "This checkpoint declares HYV4 structural tokens (think_begin_token" + "/toolcalls_begin_token/argkey_begin_token) in " + "tokenizer_config.json, which transformers 5 no longer supports. " + "Remove those fields and keep the tokens in the tokenizer's own " + "token definitions so the suffix can be read from the vocab." + ) structural_token_re = re.compile( r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>" diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 273c504f9d..0766f9b9ba 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -7,7 +7,6 @@ from collections.abc import Callable, Iterator, Sequence from contextlib import contextmanager from dataclasses import asdict from functools import cache, partial, wraps -from importlib.metadata import version from pathlib import Path from typing import Any, Literal, TypeAlias @@ -15,7 +14,6 @@ import huggingface_hub import torch import transformers.configuration_utils as hf_configuration_utils from huggingface_hub import constants -from packaging.version import Version from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE from transformers import GenerationConfig, PreTrainedConfig from transformers.configuration_utils import ALLOWED_LAYER_TYPES @@ -69,12 +67,6 @@ _DENSE_MODULE_TYPES = { "pylate.models.Dense.Dense", } -if Version(version("transformers")) < Version("5.0.0"): - raise ImportError( - "Support for Transformers v4 is deprecated and was removed in vLLM v0.24.0. " - "Please upgrade to Transformers v5: pip install --upgrade transformers" - ) - class LazyConfigDict(dict): def __getitem__(self, key): diff --git a/vllm/transformers_utils/processors/voxtral.py b/vllm/transformers_utils/processors/voxtral.py index 4cc23c4775..86c2aaf206 100644 --- a/vllm/transformers_utils/processors/voxtral.py +++ b/vllm/transformers_utils/processors/voxtral.py @@ -61,10 +61,8 @@ class MistralCommonFeatureExtractor(SequenceFeatureExtractor): """HF-compatible duck-typed ``fetch_audio``. Mirrors :meth:`transformers.SequenceFeatureExtractor.fetch_audio` so - :class:`transformers.ProcessorMixin.prepare_inputs_layout` (added in - transformers 5.10) works on this duck-typed feature extractor. Older - transformers versions never invoke this method, so the addition is a - no-op there. + :class:`transformers.ProcessorMixin.prepare_inputs_layout` works on this + duck-typed feature extractor. Accepts the same shapes as ``SequenceFeatureExtractor.fetch_audio``: @@ -77,11 +75,6 @@ class MistralCommonFeatureExtractor(SequenceFeatureExtractor): ``ProcessorMixin.prepare_inputs_layout`` always passes already-decoded audio (numpy array or torch tensor), so the str / list-of-str branches exist only to keep the contract identical to the upstream method. - - The semantics of ``transformers.audio_utils.is_valid_audio`` differ - between transformers versions (5.9 only accepts ndarray/tensor; 5.10 - also accepts ``list[float]``). We detect ``list[float]`` explicitly to - keep behavior identical across versions. """ from transformers.audio_utils import is_valid_audio @@ -89,9 +82,6 @@ class MistralCommonFeatureExtractor(SequenceFeatureExtractor): if is_valid_audio(audio_url_or_urls): return audio_url_or_urls if isinstance(audio_url_or_urls, (list, tuple)): - if audio_url_or_urls and isinstance(audio_url_or_urls[0], float): - # A single audio represented as ``list[float]``. - return audio_url_or_urls return [ self.fetch_audio(x, sampling_rate=sampling_rate) for x in audio_url_or_urls