From dd944845777b303000a2e153707382cf03383e26 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:13:28 +0100 Subject: [PATCH] Bump Transformers version to 5.10.4 (#41359) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- benchmarks/backend_request_func.py | 4 +- docs/features/reasoning_outputs.md | 2 +- .../prompt_embed/prompt_embed_offline.py | 8 +- requirements/test/cpu.txt | 4 +- requirements/test/cuda.in | 4 +- requirements/test/cuda.txt | 4 +- requirements/test/nightly-torch.txt | 2 +- requirements/test/rocm.in | 4 +- requirements/test/rocm.txt | 4 +- requirements/test/xpu.in | 1 + requirements/test/xpu.txt | 3 +- rust/src/text/src/backend/hf/model_files.rs | 2 +- tests/conftest.py | 4 +- .../entrypoints/pooling/embed/test_online.py | 4 +- .../multimodal/generation/test_common.py | 1 - .../generation/vlm_utils/model_utils.py | 91 ------------------- tests/models/registry.py | 4 +- tests/models/utils.py | 1 + tests/tokenizers_/test_basic.py | 6 +- tests/tokenizers_/test_detokenize.py | 10 +- tests/v1/engine/utils.py | 10 +- vllm/model_executor/models/funaudiochat.py | 8 +- vllm/model_executor/models/minicpmv.py | 1 + vllm/model_executor/models/minicpmv4_6.py | 31 +++++++ vllm/model_executor/models/qwen3_vl.py | 60 +++++++----- .../models/transformers/base.py | 4 +- vllm/model_executor/models/utils.py | 16 ++-- vllm/reasoning/abs_reasoning_parsers.py | 2 +- vllm/renderers/hf.py | 18 +--- vllm/tokenizers/deepseek_v32.py | 4 +- vllm/tokenizers/deepseek_v4.py | 4 +- vllm/tokenizers/hf.py | 45 ++++++++- vllm/tokenizers/mistral.py | 43 +++++++-- vllm/tokenizers/registry.py | 1 + vllm/tool_parsers/abstract_tool_parser.py | 2 +- vllm/transformers_utils/config.py | 1 + vllm/transformers_utils/configs/__init__.py | 2 + .../processors/fireredlid.py | 2 +- vllm/transformers_utils/processors/glm4v.py | 4 +- .../transformers_utils/processors/internvl.py | 11 ++- vllm/transformers_utils/processors/isaac.py | 9 +- .../transformers_utils/processors/minicpmo.py | 4 +- .../transformers_utils/processors/minicpmv.py | 15 +-- .../processors/moondream3.py | 4 +- vllm/transformers_utils/processors/openvla.py | 4 +- vllm/transformers_utils/processors/ovis.py | 4 +- vllm/transformers_utils/processors/pixtral.py | 4 +- .../transformers_utils/processors/step3_vl.py | 9 +- vllm/transformers_utils/processors/voxtral.py | 9 +- vllm/v1/engine/detokenizer.py | 6 +- vllm/v1/structured_output/backend_guidance.py | 5 + 51 files changed, 271 insertions(+), 234 deletions(-) diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py index a69637bfc4..6349095ad7 100644 --- a/benchmarks/backend_request_func.py +++ b/benchmarks/backend_request_func.py @@ -12,7 +12,7 @@ from dataclasses import dataclass, field import aiohttp import huggingface_hub.constants from tqdm.asyncio import tqdm -from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast +from transformers import AutoTokenizer, PythonBackend, TokenizersBackend # NOTE(simon): do not import vLLM here so the benchmark script # can run without vLLM installed. @@ -609,7 +609,7 @@ def get_tokenizer( tokenizer_mode: str = "auto", trust_remote_code: bool = False, **kwargs, -) -> PreTrainedTokenizer | PreTrainedTokenizerFast: +) -> PythonBackend | TokenizersBackend: if pretrained_model_name_or_path is not None and not os.path.exists( pretrained_model_name_or_path ): diff --git a/docs/features/reasoning_outputs.md b/docs/features/reasoning_outputs.md index 92563a8b4b..50a58b8b3e 100644 --- a/docs/features/reasoning_outputs.md +++ b/docs/features/reasoning_outputs.md @@ -439,7 +439,7 @@ Additionally, to enable structured output, you'll need to create a new `Reasoner end_token: str = "" @classmethod - def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner: + def from_tokenizer(cls, tokenizer: PythonBackend) -> Reasoner: return cls( start_token_id=tokenizer.encode("", add_special_tokens=False)[0], end_token_id=tokenizer.encode("", add_special_tokens=False)[0], diff --git a/examples/features/prompt_embed/prompt_embed_offline.py b/examples/features/prompt_embed/prompt_embed_offline.py index 29853bce96..9e90aa46b5 100644 --- a/examples/features/prompt_embed/prompt_embed_offline.py +++ b/examples/features/prompt_embed/prompt_embed_offline.py @@ -19,7 +19,7 @@ Run: """ import torch -from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedTokenizer +from transformers import AutoModelForCausalLM, AutoTokenizer, PythonBackend from vllm import LLM @@ -34,7 +34,7 @@ def init_tokenizer_and_llm(model_name: str): def get_prompt_embeds( chat: list[dict[str, str]], - tokenizer: PreTrainedTokenizer, + tokenizer: PythonBackend, embedding_layer: torch.nn.Module, ): token_ids = tokenizer.apply_chat_template( @@ -45,7 +45,7 @@ def get_prompt_embeds( def single_prompt_inference( - llm: LLM, tokenizer: PreTrainedTokenizer, embedding_layer: torch.nn.Module + llm: LLM, tokenizer: PythonBackend, embedding_layer: torch.nn.Module ): chat = [{"role": "user", "content": "Please tell me about the capital of France."}] prompt_embeds = get_prompt_embeds(chat, tokenizer, embedding_layer) @@ -64,7 +64,7 @@ def single_prompt_inference( def batch_prompt_inference( - llm: LLM, tokenizer: PreTrainedTokenizer, embedding_layer: torch.nn.Module + llm: LLM, tokenizer: PythonBackend, embedding_layer: torch.nn.Module ): chats = [ [{"role": "user", "content": "Please tell me about the capital of France."}], diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index 5ae281c21d..da2c7b7721 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -672,7 +672,7 @@ pathvalidate==3.2.1 # via pytablewriter patsy==1.0.1 # via statsmodels -peft==0.18.1 +peft==0.19.1 # via -r requirements/test/cuda.in perceptron==0.1.4 # via -r requirements/test/cuda.in @@ -1159,7 +1159,7 @@ tqdm==4.67.3 # segmentation-models-pytorch # sentence-transformers # transformers -transformers==5.5.3 +transformers==5.10.4 # via # -r requirements/test/../common.txt # -r requirements/test/cuda.in diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index c5518ecaa4..a0061fbf78 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -20,7 +20,7 @@ httpx librosa # required for audio tests vector_quantize_pytorch # required for minicpmo_26 test vocos # required for minicpmo_26 test -peft>=0.18.1 # required for phi-4-mm test +peft>=0.19.1 # required for phi-4-mm test pqdm ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests sentence-transformers>=5.2.0 # required for embedding tests @@ -39,7 +39,7 @@ open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_ datamodel_code_generator # required for minicpm3 test lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test -transformers==5.5.3 +transformers==5.10.4 tokenizers==0.22.2 schemathesis>=4.0.0 # Required for openai schema test. # quantization diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index 53aaa78f95..2f7b942e91 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -755,7 +755,7 @@ pathvalidate==3.2.1 # via pytablewriter patsy==1.0.1 # via statsmodels -peft==0.18.1 +peft==0.19.1 # via -r requirements/test/cuda.in perceptron==0.1.4 # via -r requirements/test/cuda.in @@ -1261,7 +1261,7 @@ tqdm==4.67.3 # segmentation-models-pytorch # sentence-transformers # transformers -transformers==5.5.3 +transformers==5.10.4 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/nightly-torch.txt b/requirements/test/nightly-torch.txt index 08f721771c..826473db1b 100644 --- a/requirements/test/nightly-torch.txt +++ b/requirements/test/nightly-torch.txt @@ -29,7 +29,7 @@ opencv-python-headless >= 4.13.0 # required for video test datamodel_code_generator # required for minicpm3 test lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test -transformers==5.5.3 +transformers==5.10.4 tokenizers==0.22.2 schemathesis>=4.0.0 # Required for openai schema test. # quantization diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index 5afa6fcec9..b1c9a473e2 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -19,7 +19,7 @@ httpx librosa # required for audio tests vector_quantize_pytorch # required for minicpmo_26 test vocos # required for minicpmo_26 test -peft>=0.15.0 # required for phi-4-mm test +peft>=0.19.1 # required for phi-4-mm test pqdm ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests sentence-transformers>=5.2.0 # required for embedding tests @@ -35,7 +35,7 @@ open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_ datamodel_code_generator # required for minicpm3 test lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test -transformers==5.5.3 +transformers==5.10.4 tokenizers==0.22.2 schemathesis>=4.0.0 # Required for openai schema test # quantization diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index e6f19b23d1..52a0ad88c9 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -731,7 +731,7 @@ pathvalidate==3.3.1 # via pytablewriter patsy==1.0.2 # via statsmodels -peft==0.18.1 +peft==0.19.1 # via -r requirements/test/rocm.in perceptron==0.1.4 # via -r requirements/test/rocm.in @@ -1218,7 +1218,7 @@ tqdm==4.67.3 # sentence-transformers # tilelang # transformers -transformers==5.5.3 +transformers==5.10.4 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/xpu.in b/requirements/test/xpu.in index b17d921d39..1172553c4a 100644 --- a/requirements/test/xpu.in +++ b/requirements/test/xpu.in @@ -17,6 +17,7 @@ accelerate arctic-inference lm_eval[api]>=0.4.12 modelscope<1.38 +transformers==5.10.4 # --- Audio Processing --- librosa diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index b5a36cf9d5..6335fc90cf 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -940,10 +940,11 @@ tqdm==4.67.3 # pqdm # sentence-transformers # transformers -transformers==5.5.3 +transformers==5.10.4 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # -r requirements/test/xpu.in # compressed-tensors # sentence-transformers # xgrammar diff --git a/rust/src/text/src/backend/hf/model_files.rs b/rust/src/text/src/backend/hf/model_files.rs index 7f84c90f39..d8ddb0139c 100644 --- a/rust/src/text/src/backend/hf/model_files.rs +++ b/rust/src/text/src/backend/hf/model_files.rs @@ -378,7 +378,7 @@ mod tests { fs::write(dir.path().join("tokenizer.json"), "{}").expect("write tokenizer"); fs::write( dir.path().join("tokenizer_config.json"), - r#"{"tokenizer_class":"PreTrainedTokenizerFast"}"#, + r#"{"tokenizer_class":"TokenizersBackend"}"#, ) .expect("write tokenizer config"); fs::write(dir.path().join("config.json"), "{}").expect("write config"); diff --git a/tests/conftest.py b/tests/conftest.py index 6f9c8fa120..94f7a83dd2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -74,7 +74,7 @@ from torch._inductor.utils import fresh_cache if TYPE_CHECKING: - from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast + from transformers import PythonBackend, TokenizersBackend from transformers.generation.utils import GenerateOutput @@ -499,7 +499,7 @@ class HfRunner: self.model = model if not skip_tokenizer_init: - self.tokenizer: "PreTrainedTokenizer | PreTrainedTokenizerFast" = ( + self.tokenizer: "PythonBackend | TokenizersBackend" = ( AutoTokenizer.from_pretrained( tokenizer_name or model_name, trust_remote_code=trust_remote_code, diff --git a/tests/entrypoints/pooling/embed/test_online.py b/tests/entrypoints/pooling/embed/test_online.py index d5565f25d3..96555ee363 100644 --- a/tests/entrypoints/pooling/embed/test_online.py +++ b/tests/entrypoints/pooling/embed/test_online.py @@ -369,7 +369,7 @@ async def test_chat_request( assert output.object == "list" assert len(output.data) == 1 assert output.model == MODEL_NAME - assert output.usage.prompt_tokens == 34 + assert output.usage.prompt_tokens == 33 # test continue_final_message response = requests.post( @@ -401,7 +401,7 @@ async def test_chat_request( assert output.object == "list" assert len(output.data) == 1 assert output.model == MODEL_NAME - assert output.usage.prompt_tokens == 36 + assert output.usage.prompt_tokens == 35 # test continue_final_message with add_generation_prompt response = requests.post( diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index ff532fd878..6c619739c6 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -906,7 +906,6 @@ VLM_TEST_SETTINGS = { max_model_len=4096, use_tokenizer_eos=True, auto_cls=AutoModelForImageTextToText, - hf_model_kwargs=model_utils.qianfan_ocr_hf_model_kwargs("baidu/Qianfan-OCR"), ), "qwen2_vl": VLMTestInfo( models=["Qwen/Qwen2-VL-2B-Instruct"], diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index 9076935e26..72151b9ff9 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -1492,94 +1492,3 @@ def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner: hf_model.model.generate = types.MethodType(_generate, hf_model.model) return hf_model - - -def qianfan_ocr_hf_model_kwargs(model_name: str) -> dict: - """Return hf_model_kwargs with a patched config for QianfanOCR.""" - from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig - - config = QianfanOCRConfig.from_pretrained(model_name) - vc = config.vision_config - if isinstance(vc.image_size, int): - vc.image_size = (vc.image_size, vc.image_size) - if isinstance(vc.patch_size, int): - vc.patch_size = (vc.patch_size, vc.patch_size) - return {"config": config} - - -def qianfan_ocr_patch_hf_runner(hf_model: HfRunner) -> HfRunner: - """Patches an HfRunner instance to run QianfanOCR model inference. - - QianfanOCR shares the same architecture as InternVLChatModel, so the - patching logic mirrors ``internvl_patch_hf_runner``. The only difference - is that we load the config via vllm's registered ``QianfanOCRConfig`` - instead of relying on ``trust_remote_code``. - """ - - class QianfanOCRProcessor: - def __init__(self, hf_runner: HfRunner): - self.tokenizer = hf_runner.tokenizer - - from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig - - self.config = QianfanOCRConfig.from_pretrained(hf_runner.model_name) - self.vision_config = self.config.vision_config - self.use_thumbnail = self.config.use_thumbnail - self.min_num = self.config.min_dynamic_patch - self.max_num = self.config.max_dynamic_patch - self.image_size = self.vision_config.image_size - - # Compute num_image_token from config instead of model attribute, - # since the transformers-native model doesn't expose it. - image_size = self.config.force_image_size or self.vision_config.image_size - patch_size = self.vision_config.patch_size - downsample_ratio = self.config.downsample_ratio - self.num_image_token = int( - (image_size // patch_size) ** 2 * (downsample_ratio**2) - ) - - def __call__( - self, - text: str, - images: PIL.Image.Image | list[PIL.Image.Image] = None, - **kwargs, - ): - from vllm.transformers_utils.processors.internvl import ( - image_to_pixel_values_internvl, - ) - - IMG_START = "" - IMG_END = "" - IMG_CONTEXT = "" - - images = [images] if isinstance(images, PIL.Image.Image) else images - pixel_values_list = [ - image_to_pixel_values_internvl( - image, - input_size=self.image_size, - min_num=self.min_num, - max_num=self.max_num, - use_thumbnail=self.use_thumbnail, - ) - for image in images - ] - num_patches_list = [pv.shape[0] for pv in pixel_values_list] - pixel_values = torch.cat(pixel_values_list, dim=0) - - for num_patches in num_patches_list: - context_tokens = IMG_CONTEXT * self.num_image_token * num_patches - image_tokens = IMG_START + context_tokens + IMG_END - text = text.replace("", image_tokens, 1) - - prompt = self.tokenizer(text, return_tensors="pt") - prompt.update({"pixel_values": pixel_values}) - return prompt - - img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("") - hf_model.model.img_context_token_id = img_context_token_id - hf_model.processor = QianfanOCRProcessor(hf_model) - hf_model.model.get_output_embeddings = ( - lambda: hf_model.model.language_model.get_output_embeddings() - ) - hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model) - return hf_model diff --git a/tests/models/registry.py b/tests/models/registry.py index b5861b4a10..85c53723ae 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -202,9 +202,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "AfmoeForCausalLM": _HfExamplesInfo("arcee-ai/Trinity-Nano-Preview"), "ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"), "ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"), - "ArcticForCausalLM": _HfExamplesInfo( - "Snowflake/snowflake-arctic-instruct", trust_remote_code=True - ), + "ArcticForCausalLM": _HfExamplesInfo("Snowflake/snowflake-arctic-instruct"), "AXK1ForCausalLM": _HfExamplesInfo("skt/A.X-K1", trust_remote_code=True), "BailingMoeForCausalLM": _HfExamplesInfo( "inclusionAI/Ling-lite-1.5", trust_remote_code=True diff --git a/tests/models/utils.py b/tests/models/utils.py index 1c4b9c93b0..91d76d5f24 100644 --- a/tests/models/utils.py +++ b/tests/models/utils.py @@ -486,6 +486,7 @@ def dummy_hf_overrides( "Gemma3nForConditionalGeneration", "Gemma4ForCausalLM", "Gemma4ForConditionalGeneration", + "Gemma4MTPModel", "DiffusionGemmaForBlockDiffusion", ) else 1 diff --git a/tests/tokenizers_/test_basic.py b/tests/tokenizers_/test_basic.py index c3e211a7ae..4da6381a0b 100644 --- a/tests/tokenizers_/test_basic.py +++ b/tests/tokenizers_/test_basic.py @@ -5,7 +5,7 @@ from typing import _get_protocol_attrs # type: ignore import pytest from transformers import ( PreTrainedTokenizerBase, - PreTrainedTokenizerFast, + TokenizersBackend, ) from vllm.tokenizers import TokenizerLike, get_tokenizer @@ -24,7 +24,7 @@ def _assert_tokenizer_like(tokenizer: object): def test_tokenizer_like_protocol(): tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True) - assert isinstance(tokenizer, PreTrainedTokenizerFast) + assert isinstance(tokenizer, TokenizersBackend) _assert_tokenizer_like(tokenizer) tokenizer = get_tokenizer( @@ -38,7 +38,7 @@ def test_tokenizer_like_protocol(): assert isinstance(tokenizer, HfTokenizer) # Verify it's a fast tokenizer (required for FastIncrementalDetokenizer) - assert isinstance(tokenizer, PreTrainedTokenizerFast) + assert isinstance(tokenizer, TokenizersBackend) assert "DSV32" in tokenizer.__class__.__name__ _assert_tokenizer_like(tokenizer) diff --git a/tests/tokenizers_/test_detokenize.py b/tests/tokenizers_/test_detokenize.py index 8244dfeb86..23eaca9fc3 100644 --- a/tests/tokenizers_/test_detokenize.py +++ b/tests/tokenizers_/test_detokenize.py @@ -5,7 +5,7 @@ from collections.abc import Generator from typing import Any import pytest -from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast +from transformers import AutoTokenizer, PythonBackend, TokenizersBackend from vllm.sampling_params import SamplingParams from vllm.tokenizers.mistral import MistralTokenizer @@ -153,13 +153,13 @@ def test_decode_streaming( spaces_between_special_tokens, fast, ): - if fast and not isinstance(tokenizer, PreTrainedTokenizerFast): + if fast and not isinstance(tokenizer, TokenizersBackend): pytest.skip() if skip_special_tokens and not spaces_between_special_tokens: pytest.skip() - if not fast and isinstance(tokenizer, PreTrainedTokenizerFast): + if not fast and isinstance(tokenizer, TokenizersBackend): # Fix up inconsistency in fast/slow tokenizer behaviour. tokenizer.add_special_tokens( { @@ -173,7 +173,7 @@ def test_decode_streaming( extra_decode_args = ( {} - if not isinstance(tokenizer, PreTrainedTokenizer) + if not isinstance(tokenizer, PythonBackend) else {"spaces_between_special_tokens": spaces_between_special_tokens} ) @@ -225,7 +225,7 @@ def test_decode_streaming( @pytest.mark.parametrize("tokenizer_name", TOKENIZERS) @pytest.mark.parametrize("fast", (True, False)) def test_oov_decode(tokenizer, fast): - if fast and not isinstance(tokenizer, PreTrainedTokenizerFast): + if fast and not isinstance(tokenizer, TokenizersBackend): pytest.skip() decoded_text, out_ids = _run_incremental_decode( diff --git a/tests/v1/engine/utils.py b/tests/v1/engine/utils.py index 013e73bd8e..324c9c9ad5 100644 --- a/tests/v1/engine/utils.py +++ b/tests/v1/engine/utils.py @@ -7,14 +7,14 @@ from typing import TypeAlias import numpy as np import torch -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast +from transformers import PythonBackend, TokenizersBackend from vllm.engine.arg_utils import EngineArgs from vllm.v1.engine import EngineCoreOutput, FinishReason from vllm.v1.metrics.stats import PrefillStats from vllm.v1.outputs import LogprobsLists, LogprobsTensors -GeneralTokenizerType: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast +GeneralTokenizerType: TypeAlias = PythonBackend | TokenizersBackend # Number of sample logprobs to request when testing sample logprobs NUM_SAMPLE_LOGPROBS_UNDER_TEST = 5 @@ -193,7 +193,7 @@ def _create_random_top_token_test_matrix( def decode_token( tok_id: int, - tokenizer: PreTrainedTokenizer, + tokenizer: PythonBackend, ) -> str: """Reproduce the process of detokenizing a token for testing purposes. @@ -210,7 +210,7 @@ def decode_token( def generate_dummy_sample_logprobs( sampled_tokens_list: list, num_logprobs: int, - tokenizer: PreTrainedTokenizer, + tokenizer: PythonBackend, ) -> list[tuple[list[int], list[float], int]]: """Generate dummy sample logprobs @@ -259,7 +259,7 @@ def generate_dummy_sample_logprobs( def generate_dummy_prompt_logprobs_tensors( prompt_tokens_list: list, num_logprobs: int, - tokenizer: PreTrainedTokenizer, + tokenizer: PythonBackend, ) -> LogprobsTensors: """Generate dummy prompt logprobs tensors diff --git a/vllm/model_executor/models/funaudiochat.py b/vllm/model_executor/models/funaudiochat.py index 9557ca6802..7e7cdcd822 100644 --- a/vllm/model_executor/models/funaudiochat.py +++ b/vllm/model_executor/models/funaudiochat.py @@ -20,7 +20,7 @@ from typing import Any import numpy as np import torch import torch.nn as nn -from transformers import PreTrainedTokenizerFast, WhisperFeatureExtractor +from transformers import TokenizersBackend, WhisperFeatureExtractor from transformers.activations import get_activation from transformers.feature_extraction_utils import BatchFeature from transformers.modeling_outputs import BaseModelOutput @@ -556,15 +556,15 @@ class FunAudioChatProcessingInfo(BaseProcessingInfo): return WhisperFeatureExtractor.from_pretrained(self.model_id) @cached_property - def speech_tokenizer(self) -> PreTrainedTokenizerFast: - return PreTrainedTokenizerFast.from_pretrained( + def speech_tokenizer(self) -> TokenizersBackend: + return TokenizersBackend.from_pretrained( self.model_id, subfolder="speech_tokenizer" ) def get_feature_extractor(self) -> WhisperFeatureExtractor: return self.feature_extractor - def get_speech_tokenizer(self) -> PreTrainedTokenizerFast: + def get_speech_tokenizer(self) -> TokenizersBackend: return self.speech_tokenizer def get_data_parser(self): diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index fa32b31560..60b1112982 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -552,6 +552,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): vendored_processor = MiniCPMVProcessor( image_processor=hf_processor.image_processor, tokenizer=hf_processor.tokenizer, + version=self.get_model_version(), ) hf_processor = vendored_processor diff --git a/vllm/model_executor/models/minicpmv4_6.py b/vllm/model_executor/models/minicpmv4_6.py index 6cdc362415..dc75928eb6 100644 --- a/vllm/model_executor/models/minicpmv4_6.py +++ b/vllm/model_executor/models/minicpmv4_6.py @@ -35,6 +35,8 @@ from vllm.multimodal.parse import ImageProcessorItems, ImageSize, VideoProcessor from vllm.multimodal.processing.processor import ( PromptReplacement, PromptUpdateDetails, + ResolvedPromptUpdate, + _seq2text, ) from vllm.sequence import IntermediateTensors @@ -408,6 +410,35 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): for modality, pattern in placeholders ] + def _recompute_cached_prompt_update( + self, cached_update: ResolvedPromptUpdate, new_item_idx: int + ) -> ResolvedPromptUpdate: + new_update = super()._recompute_cached_prompt_update( + cached_update, new_item_idx + ) + # MiniCPM-V 4.6 prefixes video placeholders with `{idx}` + # (the base class only rewrites the image modality). + if cached_update.modality == "video": + tokenizer = self.info.get_tokenizer() + id_start = getattr(tokenizer, "image_id_start_token", "") + id_end = getattr(tokenizer, "image_id_end_token", "") + video_token = getattr(tokenizer, "video_token", "<|video_pad|>") + + text = _seq2text(tokenizer, cached_update.content.full) + prev_item_idx = cached_update.item_idx + + new_update = new_update.with_content( + PromptUpdateDetails.select_text( + text.replace( + f"{id_start}{prev_item_idx}{id_end}", + f"{id_start}{new_item_idx}{id_end}", + 1, + ), + video_token, + ) + ) + return new_update + def _get_mm_fields_config( self, hf_inputs, diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index a85286164e..5252afd78e 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -33,7 +33,7 @@ import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -from transformers import BatchFeature +from transformers import BatchFeature, ProcessorMixin from transformers.models.qwen2_vl import Qwen2VLImageProcessor from transformers.models.qwen2_vl.image_processing_qwen2_vl import ( smart_resize as image_smart_resize, @@ -1191,34 +1191,31 @@ def _replace_video_token_placeholders( target: list[int], replacements: list[list[int]], ) -> list[int]: - """Replace each 3-token video placeholder with its expanded sequence. + """Replace each video placeholder with its expanded sequence. Args: prompt_ids: Token IDs of the original (unexpanded) prompt. - target: 3-element list ``[vision_start_id, video_pad_id, - vision_end_id]`` to search for. + target: Token-ID sequence to search for. New-style + (transformers>=5.10) processors expand only the bare + `[video_pad_id]`; older ones expand the full + `[vision_start_id, video_pad_id, vision_end_id]` triplet. replacements: Per-video expanded token sequences, in prompt order. Returns: - Token IDs with every placeholder triplet replaced. + Token IDs with every placeholder replaced. """ result: list[int] = [] repl_idx = 0 i = 0 n = len(prompt_ids) - t0, t1, t2 = target + k = len(target) num_repl = len(replacements) while i < n: - if ( - i + 2 < n - and prompt_ids[i] == t0 - and prompt_ids[i + 1] == t1 - and prompt_ids[i + 2] == t2 - ): + if prompt_ids[i : i + k] == target: result.extend(replacements[repl_idx]) repl_idx += 1 - i += 3 + i += k else: result.append(prompt_ids[i]) i += 1 @@ -1230,6 +1227,15 @@ def _replace_video_token_placeholders( class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]): + @staticmethod + def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool: + """Transformers>=5.10 processors override `replace_video_token` + to expand only the bare video token, keeping the prompt's outer + `<|vision_start|>`/`<|vision_end|>` markers.""" + mixin_impl = getattr(ProcessorMixin, "replace_video_token", None) + proc_impl = getattr(type(hf_processor), "replace_video_token", None) + return proc_impl is not None and proc_impl is not mixin_impl + def _call_hf_processor( self, prompt: str, @@ -1365,14 +1371,17 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) tok_kwargs=tok_kwargs, ) - # Replace each placeholder triplet with pre-computed video tokens. + # Replace each placeholder with pre-computed video tokens. if video_input_ids_lst: hf_config = self.info.get_hf_config() - video_target = [ - hf_config.vision_start_token_id, - hf_config.video_token_id, - hf_config.vision_end_token_id, - ] + if self._expands_only_video_token(self.info.get_hf_processor()): + video_target = [hf_config.video_token_id] + else: + video_target = [ + hf_config.vision_start_token_id, + hf_config.video_token_id, + hf_config.vision_end_token_id, + ] input_ids = processed_outputs.pop("input_ids") if not isinstance(input_ids, list): input_ids = input_ids.tolist() @@ -1464,17 +1473,24 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) select_token_id=select_token_id, ) + if self._expands_only_video_token(hf_processor): + # transformers>=5.10 expands only the bare video_token + video_target = hf_processor.video_token + else: + # Old-style processors expand the full placeholder + # NOTE: We match string on purpose since searching sequence of + # token ids takes more time. + video_target = "<|vision_start|><|video_pad|><|vision_end|>" + return [ PromptReplacement( modality="image", target=hf_processor.image_token, replacement=get_image_replacement_qwen3vl, ), - # NOTE: We match string on purpose since searching sequence of - # token ids takes more time. PromptReplacement( modality="video", - target="<|vision_start|><|video_pad|><|vision_end|>", + target=video_target, replacement=get_video_replacement_qwen3vl, ), ] diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py index a36ffacad9..510abe82ce 100644 --- a/vllm/model_executor/models/transformers/base.py +++ b/vllm/model_executor/models/transformers/base.py @@ -276,13 +276,13 @@ class Base( - Any quantization config specific mappings """ self.hf_to_vllm_mapper = WeightsMapper() - orig_to_new_renamings = self.hf_to_vllm_mapper.orig_to_new_renamings + orig_to_new_renaming = self.hf_to_vllm_mapper.orig_to_new_renaming orig_to_new_regex = self.hf_to_vllm_mapper.orig_to_new_regex for mapping in get_model_conversion_mapping(self.model): # Handle weights which have been renamed in Transformers if isinstance(mapping, WeightRenaming): - orig_to_new_renamings.append(mapping) + orig_to_new_renaming.append(mapping) # TODO: Handle WeightConverter to enable layer merging # Handle unexpected weights which should be ignored diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index 6f4524400c..86dc72de76 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -11,7 +11,6 @@ import regex as re import torch import torch.nn as nn from torch.nn.modules.module import register_module_module_registration_hook -from transformers import PretrainedConfig from vllm.config import VllmConfig from vllm.distributed import ( @@ -33,6 +32,9 @@ from vllm.utils.torch_utils import ( ) if TYPE_CHECKING: + from transformers import PretrainedConfig + from transformers.conversion_mapping import WeightRenaming + from vllm.model_executor.layers.quantization import QuantizationConfig logger = init_logger(__name__) @@ -46,7 +48,7 @@ class WeightsMapper: If a key maps to a value of `None`, the corresponding weight is ignored.""" - orig_to_new_renamings: list[Any] = field(default_factory=list) + orig_to_new_renaming: list["WeightRenaming"] = field(default_factory=list) orig_to_new_regex: Mapping[re.Pattern, str | None] = field(default_factory=dict) orig_to_new_substr: Mapping[str, str | None] = field(default_factory=dict) orig_to_new_stacked: Mapping[str, tuple[str, ShardId]] = field(default_factory=dict) @@ -56,9 +58,9 @@ class WeightsMapper: def __or__(self, other: "WeightsMapper") -> "WeightsMapper": """Combine two `WeightsMapper`s by merging their mappings.""" return WeightsMapper( - orig_to_new_renamings=[ - *self.orig_to_new_renamings, - *other.orig_to_new_renamings, + orig_to_new_renaming=[ + *self.orig_to_new_renaming, + *other.orig_to_new_renaming, ], orig_to_new_regex={**self.orig_to_new_regex, **other.orig_to_new_regex}, orig_to_new_substr={**self.orig_to_new_substr, **other.orig_to_new_substr}, @@ -92,7 +94,7 @@ class WeightsMapper: "k_scale to v_scale" ) - for renaming in self.orig_to_new_renamings: + for renaming in self.orig_to_new_renaming: key, _ = renaming.rename_source_key(key) for pattern, new_key in self.orig_to_new_regex.items(): @@ -426,7 +428,7 @@ def init_vllm_registered_model( vllm_config: VllmConfig, *, prefix: str = "", - hf_config: PretrainedConfig | None = None, + hf_config: "PretrainedConfig | None" = None, architectures: list[str] | None = None, ) -> nn.Module: """ diff --git a/vllm/reasoning/abs_reasoning_parsers.py b/vllm/reasoning/abs_reasoning_parsers.py index 4e28e50702..04d6a937b3 100644 --- a/vllm/reasoning/abs_reasoning_parsers.py +++ b/vllm/reasoning/abs_reasoning_parsers.py @@ -41,7 +41,7 @@ class ReasoningParser: @cached_property def vocab(self) -> dict[str, int]: - # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab + # NOTE: Only TokenizersBackend is guaranteed to have .vocab # whereas all tokenizers have .get_vocab() return self.model_tokenizer.get_vocab() diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 490f589af4..a7a6693154 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -228,20 +228,12 @@ def _try_get_processor_chat_template( if cache_key in _PROCESSOR_CHAT_TEMPLATES: return _PROCESSOR_CHAT_TEMPLATES[cache_key] - from transformers import ( - PreTrainedTokenizer, - PreTrainedTokenizerFast, - ProcessorMixin, - ) + from transformers import ProcessorMixin, PythonBackend, TokenizersBackend try: processor = cached_get_processor( tokenizer.name_or_path, - processor_cls=( - PreTrainedTokenizer, - PreTrainedTokenizerFast, - ProcessorMixin, - ), + processor_cls=(PythonBackend, TokenizersBackend, ProcessorMixin), trust_remote_code=trust_remote_code, ) if ( @@ -619,15 +611,15 @@ _cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs) @lru_cache def _get_hf_base_chat_template_params() -> frozenset[str]: - from transformers import PreTrainedTokenizer + from transformers import PythonBackend # Get standard parameters from HuggingFace's base tokenizer class. - # This dynamically extracts parameters from PreTrainedTokenizer's + # This dynamically extracts parameters from PythonBackend's # apply_chat_template method, ensuring compatibility with tokenizers # that use **kwargs to receive standard parameters. # Read signature from HF's base class - the single source of truth - base_sig = inspect.signature(PreTrainedTokenizer.apply_chat_template) + base_sig = inspect.signature(PythonBackend.apply_chat_template) # Exclude VAR_KEYWORD (**kwargs) and VAR_POSITIONAL (*args) placeholders return frozenset( diff --git a/vllm/tokenizers/deepseek_v32.py b/vllm/tokenizers/deepseek_v32.py index 51199de5c4..b388f05793 100644 --- a/vllm/tokenizers/deepseek_v32.py +++ b/vllm/tokenizers/deepseek_v32.py @@ -3,7 +3,7 @@ import copy from typing import Any -from transformers import PreTrainedTokenizerFast +from transformers import TokenizersBackend from vllm.entrypoints.chat_utils import ChatCompletionMessageParam @@ -85,5 +85,5 @@ def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: class DeepseekV32Tokenizer(TokenizerLike): @classmethod def from_pretrained(cls, *args, **kwargs) -> HfTokenizer: - tokenizer = PreTrainedTokenizerFast.from_pretrained(*args, **kwargs) + tokenizer = TokenizersBackend.from_pretrained(*args, **kwargs) return get_cached_tokenizer(get_deepseek_v32_tokenizer(tokenizer)) diff --git a/vllm/tokenizers/deepseek_v4.py b/vllm/tokenizers/deepseek_v4.py index 2a6aaaf739..3897149626 100644 --- a/vllm/tokenizers/deepseek_v4.py +++ b/vllm/tokenizers/deepseek_v4.py @@ -3,7 +3,7 @@ import copy from typing import Any -from transformers import PreTrainedTokenizerFast +from transformers import TokenizersBackend from vllm.entrypoints.chat_utils import ChatCompletionMessageParam @@ -92,5 +92,5 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: class DeepseekV4Tokenizer(TokenizerLike): @classmethod def from_pretrained(cls, *args, **kwargs) -> HfTokenizer: - tokenizer = PreTrainedTokenizerFast.from_pretrained(*args, **kwargs) + tokenizer = TokenizersBackend.from_pretrained(*args, **kwargs) return get_cached_tokenizer(get_deepseek_v4_tokenizer(tokenizer)) diff --git a/vllm/tokenizers/hf.py b/vllm/tokenizers/hf.py index 45370bbb39..bdc767acd6 100644 --- a/vllm/tokenizers/hf.py +++ b/vllm/tokenizers/hf.py @@ -6,13 +6,13 @@ import queue from pathlib import Path from typing import TypeAlias, TypeVar -from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast +from transformers import AutoTokenizer, PythonBackend, TokenizersBackend from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config from .protocol import TokenizerLike -HfTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast +HfTokenizer: TypeAlias = PythonBackend | TokenizersBackend _T = TypeVar("_T", bound=TokenizerLike) @@ -24,7 +24,7 @@ class ThreadSafeHFTokenizerMixin: def maybe_make_thread_pool(tokenizer: _T, copies: int = 1): """ - If `tokenizer` is a `PreTrainedTokenizerFast`, modify the tokenizer + If `tokenizer` is a `TokenizersBackend`, modify the tokenizer in-place to make the public interface thread-safe by routing calls through a deep-copied tokenizer pool. @@ -34,14 +34,14 @@ def maybe_make_thread_pool(tokenizer: _T, copies: int = 1): methods like ``add_special_tokens`` or ``add_tokens``. - Adjacent method calls could happen on different deep copies. """ - if not isinstance(tokenizer, PreTrainedTokenizerFast) or isinstance( + if not isinstance(tokenizer, TokenizersBackend) or isinstance( tokenizer, ThreadSafeHFTokenizerMixin ): return tokenizer og_tokenizer = copy.copy(tokenizer) - tokenizer_pool: queue.Queue[PreTrainedTokenizerFast] = queue.Queue() + tokenizer_pool: queue.Queue[TokenizersBackend] = queue.Queue() for _ in range(copies): tokenizer_pool.put(copy.deepcopy(og_tokenizer)) @@ -116,6 +116,16 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: tokenizer_all_special_tokens = tokenizer.all_special_tokens tokenizer_vocab = tokenizer.get_vocab() tokenizer_len = len(tokenizer) + # The underlying tokenizer class could be MistralCommonBackend, + # which does not implement is_fast in Transformers + tokenizer_is_fast = getattr(tokenizer, "is_fast", True) + + # MistralCommonBackend is tekken-backed and needs byte-fallback-aware tokenization. + mistral_tekkenizer = None + if getattr(getattr(tokenizer, "tokenizer", None), "instruct_tokenizer", None): + from vllm.tokenizers.mistral import mistral_common_tekkenizer + + mistral_tekkenizer = mistral_common_tekkenizer(tokenizer) max_token_id = max(tokenizer_vocab.values()) max_chars_per_token = max(len(tok) for tok in tokenizer_vocab) @@ -145,6 +155,31 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: def max_chars_per_token(self) -> int: return max_chars_per_token + @property + def is_fast(self) -> bool: + return tokenizer_is_fast + + def convert_ids_to_tokens(self, ids, skip_special_tokens: bool = False): + if mistral_tekkenizer is not None: + from vllm.tokenizers.mistral import tekken_convert_ids_to_tokens + + return tekken_convert_ids_to_tokens(mistral_tekkenizer, ids) + return super().convert_ids_to_tokens( + ids, skip_special_tokens=skip_special_tokens + ) + + def convert_tokens_to_string(self, tokens: list[str]) -> str: + if mistral_tekkenizer is not None: + from vllm.tokenizers.mistral import tekken_convert_tokens_to_string + + return tekken_convert_tokens_to_string(mistral_tekkenizer, tokens) + try: + return super().convert_tokens_to_string(tokens) + except NotImplementedError: + # The underlying tokenizer class could be MistralCommonBackend, + # which does not implement convert_tokens_to_string in Transformers + return "".join(tokens) + def get_vocab(self) -> dict[str, int]: return tokenizer_vocab diff --git a/vllm/tokenizers/mistral.py b/vllm/tokenizers/mistral.py index 8e29e1e5d6..1164f7c41a 100644 --- a/vllm/tokenizers/mistral.py +++ b/vllm/tokenizers/mistral.py @@ -188,6 +188,39 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int: return tokenizer.unk_id +def mistral_common_tekkenizer(tokenizer: object) -> "Tekkenizer | None": + """Return the underlying `Tekkenizer` for a `MistralCommonBackend`.""" + mistral = getattr(tokenizer, "tokenizer", None) + instruct = getattr(mistral, "instruct_tokenizer", None) + tekken = getattr(instruct, "tokenizer", None) + return tekken if isinstance(tekken, Tekkenizer) else None + + +def tekken_convert_ids_to_tokens( + tokenizer: "Tekkenizer", ids: Sequence[int] +) -> list[str | bytes]: + """Convert ids to pieces, using raw `bytes` for byte-fallback tokens.""" + tokens: list[str | bytes] = [tokenizer.id_to_piece(i) for i in ids] + if any("�" in t for t in tokens): + tokens = [ + tokenizer.id_to_byte_piece(i, SpecialTokenPolicy.KEEP) + if i >= tokenizer.num_special_tokens + else tokenizer.decode([i], SpecialTokenPolicy.KEEP) + for i in ids + ] + return tokens + + +def tekken_convert_tokens_to_string( + tokenizer: "Tekkenizer", tokens: Sequence[str | bytes] +) -> str: + """Reassemble pieces from `tekken_convert_ids_to_tokens` into text.""" + if any(isinstance(t, bytes) for t in tokens): + ids = [_tekken_token_to_id(tokenizer, t) for t in tokens] + return tokenizer.decode(ids, SpecialTokenPolicy.KEEP) + return "".join(cast(Sequence[str], tokens)) + + class MistralTokenizer(TokenizerLike): IS_MISTRAL_TOKENIZER = True # used by vllm.utils.mistral @@ -453,14 +486,8 @@ class MistralTokenizer(TokenizerLike): if (t in to_decode_special_tokens or t not in self._special_tokens_set) ] - if any(isinstance(t, bytes) for t in tokens): - # we need to encode and decode all tokens again - ids = [_tekken_token_to_id(self.tokenizer, t) for t in tokens] - # We filtered unwanted special tokens before - # so we can decode the rest. - decoded = self.tokenizer.decode(ids, SpecialTokenPolicy.KEEP) - else: - decoded = "".join(tokens) + # We filtered unwanted special tokens before so we can decode the rest. + decoded = tekken_convert_tokens_to_string(self.tokenizer, tokens) else: # make sure certain special tokens like Tool calls are # not decoded diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index f90e427aee..cef2f7645f 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -32,6 +32,7 @@ logger = init_logger(__name__) # - Add model type to MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS in transformers (better) # - Fix tokenizer_class on the hub for the affected models (best) _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS: set[str] = { + "internlm2", "step3_vl", "step3p7", "unlimited-ocr", diff --git a/vllm/tool_parsers/abstract_tool_parser.py b/vllm/tool_parsers/abstract_tool_parser.py index a1c4cf1ffa..acb96e28f6 100644 --- a/vllm/tool_parsers/abstract_tool_parser.py +++ b/vllm/tool_parsers/abstract_tool_parser.py @@ -112,7 +112,7 @@ class ToolParser: @cached_property def vocab(self) -> dict[str, int]: - # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab + # NOTE: Only TokenizersBackend is guaranteed to have .vocab # whereas all tokenizers have .get_vocab() return self.model_tokenizer.get_vocab() diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index ff1251665a..37b740abdb 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -71,6 +71,7 @@ class LazyConfigDict(dict): _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( afmoe="AfmoeConfig", + arctic="ArcticConfig", bagel="BagelConfig", umm="CheersConfig", chatglm="ChatGLMConfig", diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 808a8bf077..e1050d6a04 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -16,6 +16,7 @@ import importlib _CLASS_TO_MODULE: dict[str, str] = { "AfmoeConfig": "vllm.transformers_utils.configs.afmoe", + "ArcticConfig": "vllm.transformers_utils.configs.arctic", "AXK1Config": "vllm.transformers_utils.configs.AXK1", "BagelConfig": "vllm.transformers_utils.configs.bagel", "CheersConfig": "vllm.transformers_utils.configs.cheers", @@ -92,6 +93,7 @@ _CLASS_TO_MODULE: dict[str, str] = { __all__ = [ "AfmoeConfig", + "ArcticConfig", "AXK1Config", "BagelConfig", "CheersConfig", diff --git a/vllm/transformers_utils/processors/fireredlid.py b/vllm/transformers_utils/processors/fireredlid.py index cb041397d0..3afdca4c7b 100644 --- a/vllm/transformers_utils/processors/fireredlid.py +++ b/vllm/transformers_utils/processors/fireredlid.py @@ -232,7 +232,7 @@ class FireRedLIDProcessor(ProcessorMixin): """ feature_extractor_class = "FireRedLIDFeatureExtractor" - tokenizer_class = ("PreTrainedTokenizer", "PreTrainedTokenizerFast") + tokenizer_class = ("PythonBackend", "TokenizersBackend") def __init__(self, feature_extractor, tokenizer): super().__init__(feature_extractor, tokenizer) diff --git a/vllm/transformers_utils/processors/glm4v.py b/vllm/transformers_utils/processors/glm4v.py index 3ecb1bae53..a8da239552 100644 --- a/vllm/transformers_utils/processors/glm4v.py +++ b/vllm/transformers_utils/processors/glm4v.py @@ -3,7 +3,7 @@ # Adapted from # https://github.com/zai-org/CogAgent -from transformers import PreTrainedTokenizer +from transformers import PythonBackend from transformers.image_processing_utils_fast import BaseImageProcessorFast from transformers.image_utils import PILImageResampling from transformers.processing_utils import ProcessorMixin @@ -30,7 +30,7 @@ class GLM4VProcessor(ProcessorMixin): def __init__( self, image_processor: GLM4VImageProcessorFast, - tokenizer: PreTrainedTokenizer, + tokenizer: PythonBackend, ) -> None: self.image_processor = image_processor self.tokenizer = tokenizer diff --git a/vllm/transformers_utils/processors/internvl.py b/vllm/transformers_utils/processors/internvl.py index fc582deef9..22e3f5be98 100644 --- a/vllm/transformers_utils/processors/internvl.py +++ b/vllm/transformers_utils/processors/internvl.py @@ -12,7 +12,12 @@ import numpy.typing as npt import torch import torchvision.transforms as T from PIL import Image -from transformers import BatchFeature, TensorType +from transformers import ( + BaseVideoProcessor, + BatchFeature, + ImageProcessingMixin, + TensorType, +) from transformers.processing_utils import ProcessorMixin from vllm.multimodal.image import convert_image_mode @@ -215,7 +220,7 @@ def video_to_pixel_values_internvl( return pixel_values -class InternVLImageProcessor: +class InternVLImageProcessor(ImageProcessingMixin): def __init__( self, image_size: int, @@ -312,7 +317,7 @@ class InternVLImageProcessor: return BatchFeature(image_inputs, tensor_type=return_tensors) -class InternVLVideoProcessor: +class InternVLVideoProcessor(BaseVideoProcessor): def __init__( self, image_size: int, diff --git a/vllm/transformers_utils/processors/isaac.py b/vllm/transformers_utils/processors/isaac.py index da548a8f1e..2d791df016 100644 --- a/vllm/transformers_utils/processors/isaac.py +++ b/vllm/transformers_utils/processors/isaac.py @@ -7,7 +7,12 @@ import numpy as np import torch import torch.nn.functional as F from PIL import Image -from transformers import BatchFeature, ProcessorMixin, TensorType +from transformers import ( + BatchFeature, + ImageProcessingMixin, + ProcessorMixin, + TensorType, +) from transformers.processing_utils import ProcessingKwargs from typing_extensions import Unpack @@ -322,7 +327,7 @@ class IsaacProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call- } -class IsaacImageProcessor: +class IsaacImageProcessor(ImageProcessingMixin): model_input_names = ["pixel_values", "image_grid_thw"] def __init__( diff --git a/vllm/transformers_utils/processors/minicpmo.py b/vllm/transformers_utils/processors/minicpmo.py index d5e5750ca5..899e0402ba 100644 --- a/vllm/transformers_utils/processors/minicpmo.py +++ b/vllm/transformers_utils/processors/minicpmo.py @@ -268,7 +268,7 @@ class MiniCPMOProcessor(ProcessorMixin): def batch_decode(self, *args, **kwargs): """ This method forwards all its arguments to LlamaTokenizerFast's - [`~PreTrainedTokenizer.batch_decode`]. Please refer to the + [`~PythonBackend.batch_decode`]. Please refer to the docstring of this method for more information. """ output_ids = args[0] @@ -289,7 +289,7 @@ class MiniCPMOProcessor(ProcessorMixin): def decode(self, *args, **kwargs): """ This method forwards all its arguments to LlamaTokenizerFast's - [`~PreTrainedTokenizer.decode`]. Please refer to the docstring + [`~PythonBackend.decode`]. Please refer to the docstring of this method for more information. """ result = args[0] diff --git a/vllm/transformers_utils/processors/minicpmv.py b/vllm/transformers_utils/processors/minicpmv.py index 91c3a8e479..712cbbb47a 100644 --- a/vllm/transformers_utils/processors/minicpmv.py +++ b/vllm/transformers_utils/processors/minicpmv.py @@ -56,14 +56,9 @@ class MiniCPMVProcessor(ProcessorMixin): image_processor_class = "AutoImageProcessor" tokenizer_class = "AutoTokenizer" - def __init__(self, image_processor=None, tokenizer=None): + def __init__(self, image_processor=None, tokenizer=None, version=None): super().__init__(image_processor, tokenizer) - # Newer (transformers v5.7+) MiniCPM-V image processors, e.g. - # MiniCPMV4_6ImageProcessor, no longer carry a `version` attribute. - # Fall back to None instead of hard-crashing: `version` is only used - # to special-case the 2.5 tokenization path in `_convert`, and any - # value other than 2.5 takes the default branch anyway. - self.version = getattr(image_processor, "version", None) + self.version = version def __call__( self, @@ -100,7 +95,7 @@ class MiniCPMVProcessor(ProcessorMixin): def batch_decode(self, *args, **kwargs): """ This method forwards all its arguments to LlamaTokenizerFast's - [`~PreTrainedTokenizer.batch_decode`]. Please refer to the + [`~PythonBackend.batch_decode`]. Please refer to the docstring of this method for more information. """ output_ids = args[0] @@ -133,7 +128,7 @@ class MiniCPMVProcessor(ProcessorMixin): def decode(self, *args, **kwargs): """ This method forwards all its arguments to LlamaTokenizerFast's - [`~PreTrainedTokenizer.decode`]. Please refer to the docstring + [`~PythonBackend.decode`]. Please refer to the docstring of this method for more information. """ result = args[0] @@ -161,7 +156,7 @@ class MiniCPMVProcessor(ProcessorMixin): def _convert(self, input_str, max_inp_length: int | None = None): add_bos = getattr(self.tokenizer, "add_bos_token", False) - if self.version == 2.5 or add_bos: + if self.version == (2, 5) or add_bos: input_ids = self.tokenizer.encode(input_str) else: bos_id = getattr( diff --git a/vllm/transformers_utils/processors/moondream3.py b/vllm/transformers_utils/processors/moondream3.py index 289c40dd17..ae6833b200 100644 --- a/vllm/transformers_utils/processors/moondream3.py +++ b/vllm/transformers_utils/processors/moondream3.py @@ -195,7 +195,7 @@ class Moondream3Processor(ProcessorMixin): The moondream3 model uses a custom tokenizer from 'moondream/starmie-v1' instead of having tokenizer files in the model repo. """ - from transformers import AutoTokenizer, PreTrainedTokenizerFast + from transformers import AutoTokenizer, TokenizersBackend from transformers.utils import cached_file tokenizer = kwargs.pop("tokenizer", None) @@ -237,7 +237,7 @@ class Moondream3Processor(ProcessorMixin): "tokenizer.json", **cached_file_kwargs, ) - return PreTrainedTokenizerFast( + return TokenizersBackend( tokenizer_file=tokenizer_file, clean_up_tokenization_spaces=False, ) diff --git a/vllm/transformers_utils/processors/openvla.py b/vllm/transformers_utils/processors/openvla.py index 162f402383..e520f3b139 100644 --- a/vllm/transformers_utils/processors/openvla.py +++ b/vllm/transformers_utils/processors/openvla.py @@ -7,7 +7,7 @@ from typing import Any import numpy as np import torch from PIL import Image -from transformers.processing_utils import ProcessorMixin +from transformers import ImageProcessingMixin, ProcessorMixin IMAGENET_MEAN = np.array([0.484375, 0.455078125, 0.40625], dtype=np.float32) IMAGENET_STD = np.array([0.228515625, 0.2236328125, 0.224609375], dtype=np.float32) @@ -65,7 +65,7 @@ def preprocess_openvla_image(image: Any, image_size: int) -> torch.Tensor: return torch.from_numpy(pixel_values) -class OpenVLAImageProcessor: +class OpenVLAImageProcessor(ImageProcessingMixin): def __init__(self, *, image_size: int) -> None: self.image_size = image_size diff --git a/vllm/transformers_utils/processors/ovis.py b/vllm/transformers_utils/processors/ovis.py index da80f24e75..907c68035f 100644 --- a/vllm/transformers_utils/processors/ovis.py +++ b/vllm/transformers_utils/processors/ovis.py @@ -417,14 +417,14 @@ class OvisProcessor(ProcessorMixin): def batch_decode(self, *args, **kwargs): """ - This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please + This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.batch_decode`]. Please refer to the docstring of this method for more information. """ return self.tokenizer.batch_decode(*args, **kwargs) def decode(self, *args, **kwargs): """ - This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to + This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.decode`]. Please refer to the docstring of this method for more information. """ return self.tokenizer.decode(*args, **kwargs) diff --git a/vllm/transformers_utils/processors/pixtral.py b/vllm/transformers_utils/processors/pixtral.py index c03360a2a5..588ad6b7fa 100644 --- a/vllm/transformers_utils/processors/pixtral.py +++ b/vllm/transformers_utils/processors/pixtral.py @@ -4,13 +4,13 @@ import torch from mistral_common.protocol.instruct.chunk import ImageChunk from mistral_common.tokens.tokenizers.multimodal import ImageEncoder from PIL import Image -from transformers import BatchFeature, ProcessorMixin, TensorType +from transformers import BatchFeature, ImageProcessingMixin, ProcessorMixin, TensorType from transformers.image_utils import ImageInput from vllm.tokenizers.mistral import MistralTokenizer -class MistralCommonImageProcessor: +class MistralCommonImageProcessor(ImageProcessingMixin): """ Provide a HF-compatible interface for `mistral_common.tokens.tokenizers.multimodal.ImageEncoder`. diff --git a/vllm/transformers_utils/processors/step3_vl.py b/vllm/transformers_utils/processors/step3_vl.py index 71540f433f..8957a7c353 100644 --- a/vllm/transformers_utils/processors/step3_vl.py +++ b/vllm/transformers_utils/processors/step3_vl.py @@ -8,7 +8,12 @@ import torch from PIL import Image from torchvision import transforms from torchvision.transforms.functional import InterpolationMode -from transformers import BatchFeature, ProcessorMixin, TensorType +from transformers import ( + BatchFeature, + ImageProcessingMixin, + ProcessorMixin, + TensorType, +) from vllm.tokenizers import TokenizerLike @@ -240,7 +245,7 @@ class ImagePatcher: ) -class Step3VLImageProcessor: +class Step3VLImageProcessor(ImageProcessingMixin): def __init__( self, image_size: int = 728, diff --git a/vllm/transformers_utils/processors/voxtral.py b/vllm/transformers_utils/processors/voxtral.py index f67bfe9d2e..5403a7e47f 100644 --- a/vllm/transformers_utils/processors/voxtral.py +++ b/vllm/transformers_utils/processors/voxtral.py @@ -6,13 +6,18 @@ from math import ceil import numpy as np import torch from mistral_common.tokens.tokenizers.audio import AudioEncoder -from transformers import BatchFeature, ProcessorMixin, TensorType +from transformers import ( + BatchFeature, + ProcessorMixin, + SequenceFeatureExtractor, + TensorType, +) from transformers.audio_utils import AudioInput from vllm.tokenizers.mistral import MistralTokenizer -class MistralCommonFeatureExtractor: +class MistralCommonFeatureExtractor(SequenceFeatureExtractor): """ Provide a HF-compatible interface for `mistral_common.tokens.tokenizers.multimodal.AudioEncoder`. diff --git a/vllm/v1/engine/detokenizer.py b/vllm/v1/engine/detokenizer.py index 4700eecb59..50f14b9f96 100644 --- a/vllm/v1/engine/detokenizer.py +++ b/vllm/v1/engine/detokenizer.py @@ -6,7 +6,7 @@ import tokenizers import tokenizers.decoders from packaging import version from tokenizers import Tokenizer -from transformers import PreTrainedTokenizerFast +from transformers import TokenizersBackend from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike @@ -57,7 +57,7 @@ class IncrementalDetokenizer: # No tokenizer => skipping detokenization. return IncrementalDetokenizer() - if USE_FAST_DETOKENIZER and isinstance(tokenizer, PreTrainedTokenizerFast): + if USE_FAST_DETOKENIZER and isinstance(tokenizer, TokenizersBackend): # Fast tokenizer => use tokenizers library DecodeStream. return FastIncrementalDetokenizer(tokenizer, request) @@ -165,7 +165,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC): class FastIncrementalDetokenizer(BaseIncrementalDetokenizer): - def __init__(self, tokenizer: PreTrainedTokenizerFast, request: EngineCoreRequest): + def __init__(self, tokenizer: TokenizersBackend, request: EngineCoreRequest): super().__init__(request) sampling_params = request.sampling_params diff --git a/vllm/v1/structured_output/backend_guidance.py b/vllm/v1/structured_output/backend_guidance.py index 31178e9f24..19b2c76e9d 100644 --- a/vllm/v1/structured_output/backend_guidance.py +++ b/vllm/v1/structured_output/backend_guidance.py @@ -8,6 +8,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any import torch +from transformers import MistralCommonBackend from vllm.logger import init_logger from vllm.sampling_params import SamplingParams @@ -95,6 +96,10 @@ class GuidanceBackend(StructuredOutputBackend): if is_mistral_tokenizer(self.tokenizer): self.ll_tokenizer = self.tokenizer.llg_tokenizer + elif isinstance(self.tokenizer, MistralCommonBackend): + from mistral_common.guidance.tokenizer import from_mistral_tokenizer + + self.ll_tokenizer = from_mistral_tokenizer(self.tokenizer.tokenizer) else: self.ll_tokenizer = llguidance_hf.from_tokenizer( self.tokenizer, max(self.vocab_size, len(self.tokenizer))