diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py
index a69637bfc4..6349095ad7 100644
--- a/benchmarks/backend_request_func.py
+++ b/benchmarks/backend_request_func.py
@@ -12,7 +12,7 @@ from dataclasses import dataclass, field
import aiohttp
import huggingface_hub.constants
from tqdm.asyncio import tqdm
-from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
+from transformers import AutoTokenizer, PythonBackend, TokenizersBackend
# NOTE(simon): do not import vLLM here so the benchmark script
# can run without vLLM installed.
@@ -609,7 +609,7 @@ def get_tokenizer(
tokenizer_mode: str = "auto",
trust_remote_code: bool = False,
**kwargs,
-) -> PreTrainedTokenizer | PreTrainedTokenizerFast:
+) -> PythonBackend | TokenizersBackend:
if pretrained_model_name_or_path is not None and not os.path.exists(
pretrained_model_name_or_path
):
diff --git a/docs/features/reasoning_outputs.md b/docs/features/reasoning_outputs.md
index 92563a8b4b..50a58b8b3e 100644
--- a/docs/features/reasoning_outputs.md
+++ b/docs/features/reasoning_outputs.md
@@ -439,7 +439,7 @@ Additionally, to enable structured output, you'll need to create a new `Reasoner
end_token: str = ""
@classmethod
- def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner:
+ def from_tokenizer(cls, tokenizer: PythonBackend) -> Reasoner:
return cls(
start_token_id=tokenizer.encode("", add_special_tokens=False)[0],
end_token_id=tokenizer.encode("", add_special_tokens=False)[0],
diff --git a/examples/features/prompt_embed/prompt_embed_offline.py b/examples/features/prompt_embed/prompt_embed_offline.py
index 29853bce96..9e90aa46b5 100644
--- a/examples/features/prompt_embed/prompt_embed_offline.py
+++ b/examples/features/prompt_embed/prompt_embed_offline.py
@@ -19,7 +19,7 @@ Run:
"""
import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, PythonBackend
from vllm import LLM
@@ -34,7 +34,7 @@ def init_tokenizer_and_llm(model_name: str):
def get_prompt_embeds(
chat: list[dict[str, str]],
- tokenizer: PreTrainedTokenizer,
+ tokenizer: PythonBackend,
embedding_layer: torch.nn.Module,
):
token_ids = tokenizer.apply_chat_template(
@@ -45,7 +45,7 @@ def get_prompt_embeds(
def single_prompt_inference(
- llm: LLM, tokenizer: PreTrainedTokenizer, embedding_layer: torch.nn.Module
+ llm: LLM, tokenizer: PythonBackend, embedding_layer: torch.nn.Module
):
chat = [{"role": "user", "content": "Please tell me about the capital of France."}]
prompt_embeds = get_prompt_embeds(chat, tokenizer, embedding_layer)
@@ -64,7 +64,7 @@ def single_prompt_inference(
def batch_prompt_inference(
- llm: LLM, tokenizer: PreTrainedTokenizer, embedding_layer: torch.nn.Module
+ llm: LLM, tokenizer: PythonBackend, embedding_layer: torch.nn.Module
):
chats = [
[{"role": "user", "content": "Please tell me about the capital of France."}],
diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt
index 5ae281c21d..da2c7b7721 100644
--- a/requirements/test/cpu.txt
+++ b/requirements/test/cpu.txt
@@ -672,7 +672,7 @@ pathvalidate==3.2.1
# via pytablewriter
patsy==1.0.1
# via statsmodels
-peft==0.18.1
+peft==0.19.1
# via -r requirements/test/cuda.in
perceptron==0.1.4
# via -r requirements/test/cuda.in
@@ -1159,7 +1159,7 @@ tqdm==4.67.3
# segmentation-models-pytorch
# sentence-transformers
# transformers
-transformers==5.5.3
+transformers==5.10.4
# via
# -r requirements/test/../common.txt
# -r requirements/test/cuda.in
diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in
index c5518ecaa4..a0061fbf78 100644
--- a/requirements/test/cuda.in
+++ b/requirements/test/cuda.in
@@ -20,7 +20,7 @@ httpx
librosa # required for audio tests
vector_quantize_pytorch # required for minicpmo_26 test
vocos # required for minicpmo_26 test
-peft>=0.18.1 # required for phi-4-mm test
+peft>=0.19.1 # required for phi-4-mm test
pqdm
ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests
sentence-transformers>=5.2.0 # required for embedding tests
@@ -39,7 +39,7 @@ open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_
datamodel_code_generator # required for minicpm3 test
lm-eval[api]>=0.4.12 # required for model evaluation test
mteb[bm25s]>=2, <3 # required for mteb test
-transformers==5.5.3
+transformers==5.10.4
tokenizers==0.22.2
schemathesis>=4.0.0 # Required for openai schema test.
# quantization
diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt
index 53aaa78f95..2f7b942e91 100644
--- a/requirements/test/cuda.txt
+++ b/requirements/test/cuda.txt
@@ -755,7 +755,7 @@ pathvalidate==3.2.1
# via pytablewriter
patsy==1.0.1
# via statsmodels
-peft==0.18.1
+peft==0.19.1
# via -r requirements/test/cuda.in
perceptron==0.1.4
# via -r requirements/test/cuda.in
@@ -1261,7 +1261,7 @@ tqdm==4.67.3
# segmentation-models-pytorch
# sentence-transformers
# transformers
-transformers==5.5.3
+transformers==5.10.4
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
diff --git a/requirements/test/nightly-torch.txt b/requirements/test/nightly-torch.txt
index 08f721771c..826473db1b 100644
--- a/requirements/test/nightly-torch.txt
+++ b/requirements/test/nightly-torch.txt
@@ -29,7 +29,7 @@ opencv-python-headless >= 4.13.0 # required for video test
datamodel_code_generator # required for minicpm3 test
lm-eval[api]>=0.4.12 # required for model evaluation test
mteb[bm25s]>=2, <3 # required for mteb test
-transformers==5.5.3
+transformers==5.10.4
tokenizers==0.22.2
schemathesis>=4.0.0 # Required for openai schema test.
# quantization
diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in
index 5afa6fcec9..b1c9a473e2 100644
--- a/requirements/test/rocm.in
+++ b/requirements/test/rocm.in
@@ -19,7 +19,7 @@ httpx
librosa # required for audio tests
vector_quantize_pytorch # required for minicpmo_26 test
vocos # required for minicpmo_26 test
-peft>=0.15.0 # required for phi-4-mm test
+peft>=0.19.1 # required for phi-4-mm test
pqdm
ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests
sentence-transformers>=5.2.0 # required for embedding tests
@@ -35,7 +35,7 @@ open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_
datamodel_code_generator # required for minicpm3 test
lm-eval[api]>=0.4.12 # required for model evaluation test
mteb[bm25s]>=2, <3 # required for mteb test
-transformers==5.5.3
+transformers==5.10.4
tokenizers==0.22.2
schemathesis>=4.0.0 # Required for openai schema test
# quantization
diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt
index e6f19b23d1..52a0ad88c9 100644
--- a/requirements/test/rocm.txt
+++ b/requirements/test/rocm.txt
@@ -731,7 +731,7 @@ pathvalidate==3.3.1
# via pytablewriter
patsy==1.0.2
# via statsmodels
-peft==0.18.1
+peft==0.19.1
# via -r requirements/test/rocm.in
perceptron==0.1.4
# via -r requirements/test/rocm.in
@@ -1218,7 +1218,7 @@ tqdm==4.67.3
# sentence-transformers
# tilelang
# transformers
-transformers==5.5.3
+transformers==5.10.4
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
diff --git a/requirements/test/xpu.in b/requirements/test/xpu.in
index b17d921d39..1172553c4a 100644
--- a/requirements/test/xpu.in
+++ b/requirements/test/xpu.in
@@ -17,6 +17,7 @@ accelerate
arctic-inference
lm_eval[api]>=0.4.12
modelscope<1.38
+transformers==5.10.4
# --- Audio Processing ---
librosa
diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt
index b5a36cf9d5..6335fc90cf 100644
--- a/requirements/test/xpu.txt
+++ b/requirements/test/xpu.txt
@@ -940,10 +940,11 @@ tqdm==4.67.3
# pqdm
# sentence-transformers
# transformers
-transformers==5.5.3
+transformers==5.10.4
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
+ # -r requirements/test/xpu.in
# compressed-tensors
# sentence-transformers
# xgrammar
diff --git a/rust/src/text/src/backend/hf/model_files.rs b/rust/src/text/src/backend/hf/model_files.rs
index 7f84c90f39..d8ddb0139c 100644
--- a/rust/src/text/src/backend/hf/model_files.rs
+++ b/rust/src/text/src/backend/hf/model_files.rs
@@ -378,7 +378,7 @@ mod tests {
fs::write(dir.path().join("tokenizer.json"), "{}").expect("write tokenizer");
fs::write(
dir.path().join("tokenizer_config.json"),
- r#"{"tokenizer_class":"PreTrainedTokenizerFast"}"#,
+ r#"{"tokenizer_class":"TokenizersBackend"}"#,
)
.expect("write tokenizer config");
fs::write(dir.path().join("config.json"), "{}").expect("write config");
diff --git a/tests/conftest.py b/tests/conftest.py
index 6f9c8fa120..94f7a83dd2 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -74,7 +74,7 @@ from torch._inductor.utils import fresh_cache
if TYPE_CHECKING:
- from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
+ from transformers import PythonBackend, TokenizersBackend
from transformers.generation.utils import GenerateOutput
@@ -499,7 +499,7 @@ class HfRunner:
self.model = model
if not skip_tokenizer_init:
- self.tokenizer: "PreTrainedTokenizer | PreTrainedTokenizerFast" = (
+ self.tokenizer: "PythonBackend | TokenizersBackend" = (
AutoTokenizer.from_pretrained(
tokenizer_name or model_name,
trust_remote_code=trust_remote_code,
diff --git a/tests/entrypoints/pooling/embed/test_online.py b/tests/entrypoints/pooling/embed/test_online.py
index d5565f25d3..96555ee363 100644
--- a/tests/entrypoints/pooling/embed/test_online.py
+++ b/tests/entrypoints/pooling/embed/test_online.py
@@ -369,7 +369,7 @@ async def test_chat_request(
assert output.object == "list"
assert len(output.data) == 1
assert output.model == MODEL_NAME
- assert output.usage.prompt_tokens == 34
+ assert output.usage.prompt_tokens == 33
# test continue_final_message
response = requests.post(
@@ -401,7 +401,7 @@ async def test_chat_request(
assert output.object == "list"
assert len(output.data) == 1
assert output.model == MODEL_NAME
- assert output.usage.prompt_tokens == 36
+ assert output.usage.prompt_tokens == 35
# test continue_final_message with add_generation_prompt
response = requests.post(
diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py
index ff532fd878..6c619739c6 100644
--- a/tests/models/multimodal/generation/test_common.py
+++ b/tests/models/multimodal/generation/test_common.py
@@ -906,7 +906,6 @@ VLM_TEST_SETTINGS = {
max_model_len=4096,
use_tokenizer_eos=True,
auto_cls=AutoModelForImageTextToText,
- hf_model_kwargs=model_utils.qianfan_ocr_hf_model_kwargs("baidu/Qianfan-OCR"),
),
"qwen2_vl": VLMTestInfo(
models=["Qwen/Qwen2-VL-2B-Instruct"],
diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py
index 9076935e26..72151b9ff9 100644
--- a/tests/models/multimodal/generation/vlm_utils/model_utils.py
+++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py
@@ -1492,94 +1492,3 @@ def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
hf_model.model.generate = types.MethodType(_generate, hf_model.model)
return hf_model
-
-
-def qianfan_ocr_hf_model_kwargs(model_name: str) -> dict:
- """Return hf_model_kwargs with a patched config for QianfanOCR."""
- from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig
-
- config = QianfanOCRConfig.from_pretrained(model_name)
- vc = config.vision_config
- if isinstance(vc.image_size, int):
- vc.image_size = (vc.image_size, vc.image_size)
- if isinstance(vc.patch_size, int):
- vc.patch_size = (vc.patch_size, vc.patch_size)
- return {"config": config}
-
-
-def qianfan_ocr_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
- """Patches an HfRunner instance to run QianfanOCR model inference.
-
- QianfanOCR shares the same architecture as InternVLChatModel, so the
- patching logic mirrors ``internvl_patch_hf_runner``. The only difference
- is that we load the config via vllm's registered ``QianfanOCRConfig``
- instead of relying on ``trust_remote_code``.
- """
-
- class QianfanOCRProcessor:
- def __init__(self, hf_runner: HfRunner):
- self.tokenizer = hf_runner.tokenizer
-
- from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig
-
- self.config = QianfanOCRConfig.from_pretrained(hf_runner.model_name)
- self.vision_config = self.config.vision_config
- self.use_thumbnail = self.config.use_thumbnail
- self.min_num = self.config.min_dynamic_patch
- self.max_num = self.config.max_dynamic_patch
- self.image_size = self.vision_config.image_size
-
- # Compute num_image_token from config instead of model attribute,
- # since the transformers-native model doesn't expose it.
- image_size = self.config.force_image_size or self.vision_config.image_size
- patch_size = self.vision_config.patch_size
- downsample_ratio = self.config.downsample_ratio
- self.num_image_token = int(
- (image_size // patch_size) ** 2 * (downsample_ratio**2)
- )
-
- def __call__(
- self,
- text: str,
- images: PIL.Image.Image | list[PIL.Image.Image] = None,
- **kwargs,
- ):
- from vllm.transformers_utils.processors.internvl import (
- image_to_pixel_values_internvl,
- )
-
- IMG_START = "
"
- IMG_END = ""
- IMG_CONTEXT = ""
-
- images = [images] if isinstance(images, PIL.Image.Image) else images
- pixel_values_list = [
- image_to_pixel_values_internvl(
- image,
- input_size=self.image_size,
- min_num=self.min_num,
- max_num=self.max_num,
- use_thumbnail=self.use_thumbnail,
- )
- for image in images
- ]
- num_patches_list = [pv.shape[0] for pv in pixel_values_list]
- pixel_values = torch.cat(pixel_values_list, dim=0)
-
- for num_patches in num_patches_list:
- context_tokens = IMG_CONTEXT * self.num_image_token * num_patches
- image_tokens = IMG_START + context_tokens + IMG_END
- text = text.replace("", image_tokens, 1)
-
- prompt = self.tokenizer(text, return_tensors="pt")
- prompt.update({"pixel_values": pixel_values})
- return prompt
-
- img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("")
- hf_model.model.img_context_token_id = img_context_token_id
- hf_model.processor = QianfanOCRProcessor(hf_model)
- hf_model.model.get_output_embeddings = (
- lambda: hf_model.model.language_model.get_output_embeddings()
- )
- hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
- return hf_model
diff --git a/tests/models/registry.py b/tests/models/registry.py
index b5861b4a10..85c53723ae 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -202,9 +202,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"AfmoeForCausalLM": _HfExamplesInfo("arcee-ai/Trinity-Nano-Preview"),
"ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"),
"ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"),
- "ArcticForCausalLM": _HfExamplesInfo(
- "Snowflake/snowflake-arctic-instruct", trust_remote_code=True
- ),
+ "ArcticForCausalLM": _HfExamplesInfo("Snowflake/snowflake-arctic-instruct"),
"AXK1ForCausalLM": _HfExamplesInfo("skt/A.X-K1", trust_remote_code=True),
"BailingMoeForCausalLM": _HfExamplesInfo(
"inclusionAI/Ling-lite-1.5", trust_remote_code=True
diff --git a/tests/models/utils.py b/tests/models/utils.py
index 1c4b9c93b0..91d76d5f24 100644
--- a/tests/models/utils.py
+++ b/tests/models/utils.py
@@ -486,6 +486,7 @@ def dummy_hf_overrides(
"Gemma3nForConditionalGeneration",
"Gemma4ForCausalLM",
"Gemma4ForConditionalGeneration",
+ "Gemma4MTPModel",
"DiffusionGemmaForBlockDiffusion",
)
else 1
diff --git a/tests/tokenizers_/test_basic.py b/tests/tokenizers_/test_basic.py
index c3e211a7ae..4da6381a0b 100644
--- a/tests/tokenizers_/test_basic.py
+++ b/tests/tokenizers_/test_basic.py
@@ -5,7 +5,7 @@ from typing import _get_protocol_attrs # type: ignore
import pytest
from transformers import (
PreTrainedTokenizerBase,
- PreTrainedTokenizerFast,
+ TokenizersBackend,
)
from vllm.tokenizers import TokenizerLike, get_tokenizer
@@ -24,7 +24,7 @@ def _assert_tokenizer_like(tokenizer: object):
def test_tokenizer_like_protocol():
tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True)
- assert isinstance(tokenizer, PreTrainedTokenizerFast)
+ assert isinstance(tokenizer, TokenizersBackend)
_assert_tokenizer_like(tokenizer)
tokenizer = get_tokenizer(
@@ -38,7 +38,7 @@ def test_tokenizer_like_protocol():
assert isinstance(tokenizer, HfTokenizer)
# Verify it's a fast tokenizer (required for FastIncrementalDetokenizer)
- assert isinstance(tokenizer, PreTrainedTokenizerFast)
+ assert isinstance(tokenizer, TokenizersBackend)
assert "DSV32" in tokenizer.__class__.__name__
_assert_tokenizer_like(tokenizer)
diff --git a/tests/tokenizers_/test_detokenize.py b/tests/tokenizers_/test_detokenize.py
index 8244dfeb86..23eaca9fc3 100644
--- a/tests/tokenizers_/test_detokenize.py
+++ b/tests/tokenizers_/test_detokenize.py
@@ -5,7 +5,7 @@ from collections.abc import Generator
from typing import Any
import pytest
-from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
+from transformers import AutoTokenizer, PythonBackend, TokenizersBackend
from vllm.sampling_params import SamplingParams
from vllm.tokenizers.mistral import MistralTokenizer
@@ -153,13 +153,13 @@ def test_decode_streaming(
spaces_between_special_tokens,
fast,
):
- if fast and not isinstance(tokenizer, PreTrainedTokenizerFast):
+ if fast and not isinstance(tokenizer, TokenizersBackend):
pytest.skip()
if skip_special_tokens and not spaces_between_special_tokens:
pytest.skip()
- if not fast and isinstance(tokenizer, PreTrainedTokenizerFast):
+ if not fast and isinstance(tokenizer, TokenizersBackend):
# Fix up inconsistency in fast/slow tokenizer behaviour.
tokenizer.add_special_tokens(
{
@@ -173,7 +173,7 @@ def test_decode_streaming(
extra_decode_args = (
{}
- if not isinstance(tokenizer, PreTrainedTokenizer)
+ if not isinstance(tokenizer, PythonBackend)
else {"spaces_between_special_tokens": spaces_between_special_tokens}
)
@@ -225,7 +225,7 @@ def test_decode_streaming(
@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
@pytest.mark.parametrize("fast", (True, False))
def test_oov_decode(tokenizer, fast):
- if fast and not isinstance(tokenizer, PreTrainedTokenizerFast):
+ if fast and not isinstance(tokenizer, TokenizersBackend):
pytest.skip()
decoded_text, out_ids = _run_incremental_decode(
diff --git a/tests/v1/engine/utils.py b/tests/v1/engine/utils.py
index 013e73bd8e..324c9c9ad5 100644
--- a/tests/v1/engine/utils.py
+++ b/tests/v1/engine/utils.py
@@ -7,14 +7,14 @@ from typing import TypeAlias
import numpy as np
import torch
-from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
+from transformers import PythonBackend, TokenizersBackend
from vllm.engine.arg_utils import EngineArgs
from vllm.v1.engine import EngineCoreOutput, FinishReason
from vllm.v1.metrics.stats import PrefillStats
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
-GeneralTokenizerType: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast
+GeneralTokenizerType: TypeAlias = PythonBackend | TokenizersBackend
# Number of sample logprobs to request when testing sample logprobs
NUM_SAMPLE_LOGPROBS_UNDER_TEST = 5
@@ -193,7 +193,7 @@ def _create_random_top_token_test_matrix(
def decode_token(
tok_id: int,
- tokenizer: PreTrainedTokenizer,
+ tokenizer: PythonBackend,
) -> str:
"""Reproduce the process of detokenizing a token for testing purposes.
@@ -210,7 +210,7 @@ def decode_token(
def generate_dummy_sample_logprobs(
sampled_tokens_list: list,
num_logprobs: int,
- tokenizer: PreTrainedTokenizer,
+ tokenizer: PythonBackend,
) -> list[tuple[list[int], list[float], int]]:
"""Generate dummy sample logprobs
@@ -259,7 +259,7 @@ def generate_dummy_sample_logprobs(
def generate_dummy_prompt_logprobs_tensors(
prompt_tokens_list: list,
num_logprobs: int,
- tokenizer: PreTrainedTokenizer,
+ tokenizer: PythonBackend,
) -> LogprobsTensors:
"""Generate dummy prompt logprobs tensors
diff --git a/vllm/model_executor/models/funaudiochat.py b/vllm/model_executor/models/funaudiochat.py
index 9557ca6802..7e7cdcd822 100644
--- a/vllm/model_executor/models/funaudiochat.py
+++ b/vllm/model_executor/models/funaudiochat.py
@@ -20,7 +20,7 @@ from typing import Any
import numpy as np
import torch
import torch.nn as nn
-from transformers import PreTrainedTokenizerFast, WhisperFeatureExtractor
+from transformers import TokenizersBackend, WhisperFeatureExtractor
from transformers.activations import get_activation
from transformers.feature_extraction_utils import BatchFeature
from transformers.modeling_outputs import BaseModelOutput
@@ -556,15 +556,15 @@ class FunAudioChatProcessingInfo(BaseProcessingInfo):
return WhisperFeatureExtractor.from_pretrained(self.model_id)
@cached_property
- def speech_tokenizer(self) -> PreTrainedTokenizerFast:
- return PreTrainedTokenizerFast.from_pretrained(
+ def speech_tokenizer(self) -> TokenizersBackend:
+ return TokenizersBackend.from_pretrained(
self.model_id, subfolder="speech_tokenizer"
)
def get_feature_extractor(self) -> WhisperFeatureExtractor:
return self.feature_extractor
- def get_speech_tokenizer(self) -> PreTrainedTokenizerFast:
+ def get_speech_tokenizer(self) -> TokenizersBackend:
return self.speech_tokenizer
def get_data_parser(self):
diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py
index fa32b31560..60b1112982 100644
--- a/vllm/model_executor/models/minicpmv.py
+++ b/vllm/model_executor/models/minicpmv.py
@@ -552,6 +552,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
vendored_processor = MiniCPMVProcessor(
image_processor=hf_processor.image_processor,
tokenizer=hf_processor.tokenizer,
+ version=self.get_model_version(),
)
hf_processor = vendored_processor
diff --git a/vllm/model_executor/models/minicpmv4_6.py b/vllm/model_executor/models/minicpmv4_6.py
index 6cdc362415..dc75928eb6 100644
--- a/vllm/model_executor/models/minicpmv4_6.py
+++ b/vllm/model_executor/models/minicpmv4_6.py
@@ -35,6 +35,8 @@ from vllm.multimodal.parse import ImageProcessorItems, ImageSize, VideoProcessor
from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdateDetails,
+ ResolvedPromptUpdate,
+ _seq2text,
)
from vllm.sequence import IntermediateTensors
@@ -408,6 +410,35 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
for modality, pattern in placeholders
]
+ def _recompute_cached_prompt_update(
+ self, cached_update: ResolvedPromptUpdate, new_item_idx: int
+ ) -> ResolvedPromptUpdate:
+ new_update = super()._recompute_cached_prompt_update(
+ cached_update, new_item_idx
+ )
+ # MiniCPM-V 4.6 prefixes video placeholders with `{idx}`
+ # (the base class only rewrites the image modality).
+ if cached_update.modality == "video":
+ tokenizer = self.info.get_tokenizer()
+ id_start = getattr(tokenizer, "image_id_start_token", "")
+ id_end = getattr(tokenizer, "image_id_end_token", "")
+ video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
+
+ text = _seq2text(tokenizer, cached_update.content.full)
+ prev_item_idx = cached_update.item_idx
+
+ new_update = new_update.with_content(
+ PromptUpdateDetails.select_text(
+ text.replace(
+ f"{id_start}{prev_item_idx}{id_end}",
+ f"{id_start}{new_item_idx}{id_end}",
+ 1,
+ ),
+ video_token,
+ )
+ )
+ return new_update
+
def _get_mm_fields_config(
self,
hf_inputs,
diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py
index a85286164e..5252afd78e 100644
--- a/vllm/model_executor/models/qwen3_vl.py
+++ b/vllm/model_executor/models/qwen3_vl.py
@@ -33,7 +33,7 @@ import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
-from transformers import BatchFeature
+from transformers import BatchFeature, ProcessorMixin
from transformers.models.qwen2_vl import Qwen2VLImageProcessor
from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
smart_resize as image_smart_resize,
@@ -1191,34 +1191,31 @@ def _replace_video_token_placeholders(
target: list[int],
replacements: list[list[int]],
) -> list[int]:
- """Replace each 3-token video placeholder with its expanded sequence.
+ """Replace each video placeholder with its expanded sequence.
Args:
prompt_ids: Token IDs of the original (unexpanded) prompt.
- target: 3-element list ``[vision_start_id, video_pad_id,
- vision_end_id]`` to search for.
+ target: Token-ID sequence to search for. New-style
+ (transformers>=5.10) processors expand only the bare
+ `[video_pad_id]`; older ones expand the full
+ `[vision_start_id, video_pad_id, vision_end_id]` triplet.
replacements: Per-video expanded token sequences, in prompt order.
Returns:
- Token IDs with every placeholder triplet replaced.
+ Token IDs with every placeholder replaced.
"""
result: list[int] = []
repl_idx = 0
i = 0
n = len(prompt_ids)
- t0, t1, t2 = target
+ k = len(target)
num_repl = len(replacements)
while i < n:
- if (
- i + 2 < n
- and prompt_ids[i] == t0
- and prompt_ids[i + 1] == t1
- and prompt_ids[i + 2] == t2
- ):
+ if prompt_ids[i : i + k] == target:
result.extend(replacements[repl_idx])
repl_idx += 1
- i += 3
+ i += k
else:
result.append(prompt_ids[i])
i += 1
@@ -1230,6 +1227,15 @@ def _replace_video_token_placeholders(
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
+ @staticmethod
+ def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
+ """Transformers>=5.10 processors override `replace_video_token`
+ to expand only the bare video token, keeping the prompt's outer
+ `<|vision_start|>`/`<|vision_end|>` markers."""
+ mixin_impl = getattr(ProcessorMixin, "replace_video_token", None)
+ proc_impl = getattr(type(hf_processor), "replace_video_token", None)
+ return proc_impl is not None and proc_impl is not mixin_impl
+
def _call_hf_processor(
self,
prompt: str,
@@ -1365,14 +1371,17 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
tok_kwargs=tok_kwargs,
)
- # Replace each placeholder triplet with pre-computed video tokens.
+ # Replace each placeholder with pre-computed video tokens.
if video_input_ids_lst:
hf_config = self.info.get_hf_config()
- video_target = [
- hf_config.vision_start_token_id,
- hf_config.video_token_id,
- hf_config.vision_end_token_id,
- ]
+ if self._expands_only_video_token(self.info.get_hf_processor()):
+ video_target = [hf_config.video_token_id]
+ else:
+ video_target = [
+ hf_config.vision_start_token_id,
+ hf_config.video_token_id,
+ hf_config.vision_end_token_id,
+ ]
input_ids = processed_outputs.pop("input_ids")
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
@@ -1464,17 +1473,24 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
select_token_id=select_token_id,
)
+ if self._expands_only_video_token(hf_processor):
+ # transformers>=5.10 expands only the bare video_token
+ video_target = hf_processor.video_token
+ else:
+ # Old-style processors expand the full placeholder
+ # NOTE: We match string on purpose since searching sequence of
+ # token ids takes more time.
+ video_target = "<|vision_start|><|video_pad|><|vision_end|>"
+
return [
PromptReplacement(
modality="image",
target=hf_processor.image_token,
replacement=get_image_replacement_qwen3vl,
),
- # NOTE: We match string on purpose since searching sequence of
- # token ids takes more time.
PromptReplacement(
modality="video",
- target="<|vision_start|><|video_pad|><|vision_end|>",
+ target=video_target,
replacement=get_video_replacement_qwen3vl,
),
]
diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py
index a36ffacad9..510abe82ce 100644
--- a/vllm/model_executor/models/transformers/base.py
+++ b/vllm/model_executor/models/transformers/base.py
@@ -276,13 +276,13 @@ class Base(
- Any quantization config specific mappings
"""
self.hf_to_vllm_mapper = WeightsMapper()
- orig_to_new_renamings = self.hf_to_vllm_mapper.orig_to_new_renamings
+ orig_to_new_renaming = self.hf_to_vllm_mapper.orig_to_new_renaming
orig_to_new_regex = self.hf_to_vllm_mapper.orig_to_new_regex
for mapping in get_model_conversion_mapping(self.model):
# Handle weights which have been renamed in Transformers
if isinstance(mapping, WeightRenaming):
- orig_to_new_renamings.append(mapping)
+ orig_to_new_renaming.append(mapping)
# TODO: Handle WeightConverter to enable layer merging
# Handle unexpected weights which should be ignored
diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py
index 6f4524400c..86dc72de76 100644
--- a/vllm/model_executor/models/utils.py
+++ b/vllm/model_executor/models/utils.py
@@ -11,7 +11,6 @@ import regex as re
import torch
import torch.nn as nn
from torch.nn.modules.module import register_module_module_registration_hook
-from transformers import PretrainedConfig
from vllm.config import VllmConfig
from vllm.distributed import (
@@ -33,6 +32,9 @@ from vllm.utils.torch_utils import (
)
if TYPE_CHECKING:
+ from transformers import PretrainedConfig
+ from transformers.conversion_mapping import WeightRenaming
+
from vllm.model_executor.layers.quantization import QuantizationConfig
logger = init_logger(__name__)
@@ -46,7 +48,7 @@ class WeightsMapper:
If a key maps to a value of `None`, the corresponding weight is ignored."""
- orig_to_new_renamings: list[Any] = field(default_factory=list)
+ orig_to_new_renaming: list["WeightRenaming"] = field(default_factory=list)
orig_to_new_regex: Mapping[re.Pattern, str | None] = field(default_factory=dict)
orig_to_new_substr: Mapping[str, str | None] = field(default_factory=dict)
orig_to_new_stacked: Mapping[str, tuple[str, ShardId]] = field(default_factory=dict)
@@ -56,9 +58,9 @@ class WeightsMapper:
def __or__(self, other: "WeightsMapper") -> "WeightsMapper":
"""Combine two `WeightsMapper`s by merging their mappings."""
return WeightsMapper(
- orig_to_new_renamings=[
- *self.orig_to_new_renamings,
- *other.orig_to_new_renamings,
+ orig_to_new_renaming=[
+ *self.orig_to_new_renaming,
+ *other.orig_to_new_renaming,
],
orig_to_new_regex={**self.orig_to_new_regex, **other.orig_to_new_regex},
orig_to_new_substr={**self.orig_to_new_substr, **other.orig_to_new_substr},
@@ -92,7 +94,7 @@ class WeightsMapper:
"k_scale to v_scale"
)
- for renaming in self.orig_to_new_renamings:
+ for renaming in self.orig_to_new_renaming:
key, _ = renaming.rename_source_key(key)
for pattern, new_key in self.orig_to_new_regex.items():
@@ -426,7 +428,7 @@ def init_vllm_registered_model(
vllm_config: VllmConfig,
*,
prefix: str = "",
- hf_config: PretrainedConfig | None = None,
+ hf_config: "PretrainedConfig | None" = None,
architectures: list[str] | None = None,
) -> nn.Module:
"""
diff --git a/vllm/reasoning/abs_reasoning_parsers.py b/vllm/reasoning/abs_reasoning_parsers.py
index 4e28e50702..04d6a937b3 100644
--- a/vllm/reasoning/abs_reasoning_parsers.py
+++ b/vllm/reasoning/abs_reasoning_parsers.py
@@ -41,7 +41,7 @@ class ReasoningParser:
@cached_property
def vocab(self) -> dict[str, int]:
- # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab
+ # NOTE: Only TokenizersBackend is guaranteed to have .vocab
# whereas all tokenizers have .get_vocab()
return self.model_tokenizer.get_vocab()
diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py
index 490f589af4..a7a6693154 100644
--- a/vllm/renderers/hf.py
+++ b/vllm/renderers/hf.py
@@ -228,20 +228,12 @@ def _try_get_processor_chat_template(
if cache_key in _PROCESSOR_CHAT_TEMPLATES:
return _PROCESSOR_CHAT_TEMPLATES[cache_key]
- from transformers import (
- PreTrainedTokenizer,
- PreTrainedTokenizerFast,
- ProcessorMixin,
- )
+ from transformers import ProcessorMixin, PythonBackend, TokenizersBackend
try:
processor = cached_get_processor(
tokenizer.name_or_path,
- processor_cls=(
- PreTrainedTokenizer,
- PreTrainedTokenizerFast,
- ProcessorMixin,
- ),
+ processor_cls=(PythonBackend, TokenizersBackend, ProcessorMixin),
trust_remote_code=trust_remote_code,
)
if (
@@ -619,15 +611,15 @@ _cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs)
@lru_cache
def _get_hf_base_chat_template_params() -> frozenset[str]:
- from transformers import PreTrainedTokenizer
+ from transformers import PythonBackend
# Get standard parameters from HuggingFace's base tokenizer class.
- # This dynamically extracts parameters from PreTrainedTokenizer's
+ # This dynamically extracts parameters from PythonBackend's
# apply_chat_template method, ensuring compatibility with tokenizers
# that use **kwargs to receive standard parameters.
# Read signature from HF's base class - the single source of truth
- base_sig = inspect.signature(PreTrainedTokenizer.apply_chat_template)
+ base_sig = inspect.signature(PythonBackend.apply_chat_template)
# Exclude VAR_KEYWORD (**kwargs) and VAR_POSITIONAL (*args) placeholders
return frozenset(
diff --git a/vllm/tokenizers/deepseek_v32.py b/vllm/tokenizers/deepseek_v32.py
index 51199de5c4..b388f05793 100644
--- a/vllm/tokenizers/deepseek_v32.py
+++ b/vllm/tokenizers/deepseek_v32.py
@@ -3,7 +3,7 @@
import copy
from typing import Any
-from transformers import PreTrainedTokenizerFast
+from transformers import TokenizersBackend
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
@@ -85,5 +85,5 @@ def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
class DeepseekV32Tokenizer(TokenizerLike):
@classmethod
def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:
- tokenizer = PreTrainedTokenizerFast.from_pretrained(*args, **kwargs)
+ tokenizer = TokenizersBackend.from_pretrained(*args, **kwargs)
return get_cached_tokenizer(get_deepseek_v32_tokenizer(tokenizer))
diff --git a/vllm/tokenizers/deepseek_v4.py b/vllm/tokenizers/deepseek_v4.py
index 2a6aaaf739..3897149626 100644
--- a/vllm/tokenizers/deepseek_v4.py
+++ b/vllm/tokenizers/deepseek_v4.py
@@ -3,7 +3,7 @@
import copy
from typing import Any
-from transformers import PreTrainedTokenizerFast
+from transformers import TokenizersBackend
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
@@ -92,5 +92,5 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
class DeepseekV4Tokenizer(TokenizerLike):
@classmethod
def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:
- tokenizer = PreTrainedTokenizerFast.from_pretrained(*args, **kwargs)
+ tokenizer = TokenizersBackend.from_pretrained(*args, **kwargs)
return get_cached_tokenizer(get_deepseek_v4_tokenizer(tokenizer))
diff --git a/vllm/tokenizers/hf.py b/vllm/tokenizers/hf.py
index 45370bbb39..bdc767acd6 100644
--- a/vllm/tokenizers/hf.py
+++ b/vllm/tokenizers/hf.py
@@ -6,13 +6,13 @@ import queue
from pathlib import Path
from typing import TypeAlias, TypeVar
-from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
+from transformers import AutoTokenizer, PythonBackend, TokenizersBackend
from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config
from .protocol import TokenizerLike
-HfTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast
+HfTokenizer: TypeAlias = PythonBackend | TokenizersBackend
_T = TypeVar("_T", bound=TokenizerLike)
@@ -24,7 +24,7 @@ class ThreadSafeHFTokenizerMixin:
def maybe_make_thread_pool(tokenizer: _T, copies: int = 1):
"""
- If `tokenizer` is a `PreTrainedTokenizerFast`, modify the tokenizer
+ If `tokenizer` is a `TokenizersBackend`, modify the tokenizer
in-place to make the public interface thread-safe by routing calls
through a deep-copied tokenizer pool.
@@ -34,14 +34,14 @@ def maybe_make_thread_pool(tokenizer: _T, copies: int = 1):
methods like ``add_special_tokens`` or ``add_tokens``.
- Adjacent method calls could happen on different deep copies.
"""
- if not isinstance(tokenizer, PreTrainedTokenizerFast) or isinstance(
+ if not isinstance(tokenizer, TokenizersBackend) or isinstance(
tokenizer, ThreadSafeHFTokenizerMixin
):
return tokenizer
og_tokenizer = copy.copy(tokenizer)
- tokenizer_pool: queue.Queue[PreTrainedTokenizerFast] = queue.Queue()
+ tokenizer_pool: queue.Queue[TokenizersBackend] = queue.Queue()
for _ in range(copies):
tokenizer_pool.put(copy.deepcopy(og_tokenizer))
@@ -116,6 +116,16 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
tokenizer_all_special_tokens = tokenizer.all_special_tokens
tokenizer_vocab = tokenizer.get_vocab()
tokenizer_len = len(tokenizer)
+ # The underlying tokenizer class could be MistralCommonBackend,
+ # which does not implement is_fast in Transformers
+ tokenizer_is_fast = getattr(tokenizer, "is_fast", True)
+
+ # MistralCommonBackend is tekken-backed and needs byte-fallback-aware tokenization.
+ mistral_tekkenizer = None
+ if getattr(getattr(tokenizer, "tokenizer", None), "instruct_tokenizer", None):
+ from vllm.tokenizers.mistral import mistral_common_tekkenizer
+
+ mistral_tekkenizer = mistral_common_tekkenizer(tokenizer)
max_token_id = max(tokenizer_vocab.values())
max_chars_per_token = max(len(tok) for tok in tokenizer_vocab)
@@ -145,6 +155,31 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
def max_chars_per_token(self) -> int:
return max_chars_per_token
+ @property
+ def is_fast(self) -> bool:
+ return tokenizer_is_fast
+
+ def convert_ids_to_tokens(self, ids, skip_special_tokens: bool = False):
+ if mistral_tekkenizer is not None:
+ from vllm.tokenizers.mistral import tekken_convert_ids_to_tokens
+
+ return tekken_convert_ids_to_tokens(mistral_tekkenizer, ids)
+ return super().convert_ids_to_tokens(
+ ids, skip_special_tokens=skip_special_tokens
+ )
+
+ def convert_tokens_to_string(self, tokens: list[str]) -> str:
+ if mistral_tekkenizer is not None:
+ from vllm.tokenizers.mistral import tekken_convert_tokens_to_string
+
+ return tekken_convert_tokens_to_string(mistral_tekkenizer, tokens)
+ try:
+ return super().convert_tokens_to_string(tokens)
+ except NotImplementedError:
+ # The underlying tokenizer class could be MistralCommonBackend,
+ # which does not implement convert_tokens_to_string in Transformers
+ return "".join(tokens)
+
def get_vocab(self) -> dict[str, int]:
return tokenizer_vocab
diff --git a/vllm/tokenizers/mistral.py b/vllm/tokenizers/mistral.py
index 8e29e1e5d6..1164f7c41a 100644
--- a/vllm/tokenizers/mistral.py
+++ b/vllm/tokenizers/mistral.py
@@ -188,6 +188,39 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int:
return tokenizer.unk_id
+def mistral_common_tekkenizer(tokenizer: object) -> "Tekkenizer | None":
+ """Return the underlying `Tekkenizer` for a `MistralCommonBackend`."""
+ mistral = getattr(tokenizer, "tokenizer", None)
+ instruct = getattr(mistral, "instruct_tokenizer", None)
+ tekken = getattr(instruct, "tokenizer", None)
+ return tekken if isinstance(tekken, Tekkenizer) else None
+
+
+def tekken_convert_ids_to_tokens(
+ tokenizer: "Tekkenizer", ids: Sequence[int]
+) -> list[str | bytes]:
+ """Convert ids to pieces, using raw `bytes` for byte-fallback tokens."""
+ tokens: list[str | bytes] = [tokenizer.id_to_piece(i) for i in ids]
+ if any("�" in t for t in tokens):
+ tokens = [
+ tokenizer.id_to_byte_piece(i, SpecialTokenPolicy.KEEP)
+ if i >= tokenizer.num_special_tokens
+ else tokenizer.decode([i], SpecialTokenPolicy.KEEP)
+ for i in ids
+ ]
+ return tokens
+
+
+def tekken_convert_tokens_to_string(
+ tokenizer: "Tekkenizer", tokens: Sequence[str | bytes]
+) -> str:
+ """Reassemble pieces from `tekken_convert_ids_to_tokens` into text."""
+ if any(isinstance(t, bytes) for t in tokens):
+ ids = [_tekken_token_to_id(tokenizer, t) for t in tokens]
+ return tokenizer.decode(ids, SpecialTokenPolicy.KEEP)
+ return "".join(cast(Sequence[str], tokens))
+
+
class MistralTokenizer(TokenizerLike):
IS_MISTRAL_TOKENIZER = True # used by vllm.utils.mistral
@@ -453,14 +486,8 @@ class MistralTokenizer(TokenizerLike):
if (t in to_decode_special_tokens or t not in self._special_tokens_set)
]
- if any(isinstance(t, bytes) for t in tokens):
- # we need to encode and decode all tokens again
- ids = [_tekken_token_to_id(self.tokenizer, t) for t in tokens]
- # We filtered unwanted special tokens before
- # so we can decode the rest.
- decoded = self.tokenizer.decode(ids, SpecialTokenPolicy.KEEP)
- else:
- decoded = "".join(tokens)
+ # We filtered unwanted special tokens before so we can decode the rest.
+ decoded = tekken_convert_tokens_to_string(self.tokenizer, tokens)
else:
# make sure certain special tokens like Tool calls are
# not decoded
diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py
index f90e427aee..cef2f7645f 100644
--- a/vllm/tokenizers/registry.py
+++ b/vllm/tokenizers/registry.py
@@ -32,6 +32,7 @@ logger = init_logger(__name__)
# - Add model type to MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS in transformers (better)
# - Fix tokenizer_class on the hub for the affected models (best)
_MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS: set[str] = {
+ "internlm2",
"step3_vl",
"step3p7",
"unlimited-ocr",
diff --git a/vllm/tool_parsers/abstract_tool_parser.py b/vllm/tool_parsers/abstract_tool_parser.py
index a1c4cf1ffa..acb96e28f6 100644
--- a/vllm/tool_parsers/abstract_tool_parser.py
+++ b/vllm/tool_parsers/abstract_tool_parser.py
@@ -112,7 +112,7 @@ class ToolParser:
@cached_property
def vocab(self) -> dict[str, int]:
- # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab
+ # NOTE: Only TokenizersBackend is guaranteed to have .vocab
# whereas all tokenizers have .get_vocab()
return self.model_tokenizer.get_vocab()
diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py
index ff1251665a..37b740abdb 100644
--- a/vllm/transformers_utils/config.py
+++ b/vllm/transformers_utils/config.py
@@ -71,6 +71,7 @@ class LazyConfigDict(dict):
_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
afmoe="AfmoeConfig",
+ arctic="ArcticConfig",
bagel="BagelConfig",
umm="CheersConfig",
chatglm="ChatGLMConfig",
diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py
index 808a8bf077..e1050d6a04 100644
--- a/vllm/transformers_utils/configs/__init__.py
+++ b/vllm/transformers_utils/configs/__init__.py
@@ -16,6 +16,7 @@ import importlib
_CLASS_TO_MODULE: dict[str, str] = {
"AfmoeConfig": "vllm.transformers_utils.configs.afmoe",
+ "ArcticConfig": "vllm.transformers_utils.configs.arctic",
"AXK1Config": "vllm.transformers_utils.configs.AXK1",
"BagelConfig": "vllm.transformers_utils.configs.bagel",
"CheersConfig": "vllm.transformers_utils.configs.cheers",
@@ -92,6 +93,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
__all__ = [
"AfmoeConfig",
+ "ArcticConfig",
"AXK1Config",
"BagelConfig",
"CheersConfig",
diff --git a/vllm/transformers_utils/processors/fireredlid.py b/vllm/transformers_utils/processors/fireredlid.py
index cb041397d0..3afdca4c7b 100644
--- a/vllm/transformers_utils/processors/fireredlid.py
+++ b/vllm/transformers_utils/processors/fireredlid.py
@@ -232,7 +232,7 @@ class FireRedLIDProcessor(ProcessorMixin):
"""
feature_extractor_class = "FireRedLIDFeatureExtractor"
- tokenizer_class = ("PreTrainedTokenizer", "PreTrainedTokenizerFast")
+ tokenizer_class = ("PythonBackend", "TokenizersBackend")
def __init__(self, feature_extractor, tokenizer):
super().__init__(feature_extractor, tokenizer)
diff --git a/vllm/transformers_utils/processors/glm4v.py b/vllm/transformers_utils/processors/glm4v.py
index 3ecb1bae53..a8da239552 100644
--- a/vllm/transformers_utils/processors/glm4v.py
+++ b/vllm/transformers_utils/processors/glm4v.py
@@ -3,7 +3,7 @@
# Adapted from
# https://github.com/zai-org/CogAgent
-from transformers import PreTrainedTokenizer
+from transformers import PythonBackend
from transformers.image_processing_utils_fast import BaseImageProcessorFast
from transformers.image_utils import PILImageResampling
from transformers.processing_utils import ProcessorMixin
@@ -30,7 +30,7 @@ class GLM4VProcessor(ProcessorMixin):
def __init__(
self,
image_processor: GLM4VImageProcessorFast,
- tokenizer: PreTrainedTokenizer,
+ tokenizer: PythonBackend,
) -> None:
self.image_processor = image_processor
self.tokenizer = tokenizer
diff --git a/vllm/transformers_utils/processors/internvl.py b/vllm/transformers_utils/processors/internvl.py
index fc582deef9..22e3f5be98 100644
--- a/vllm/transformers_utils/processors/internvl.py
+++ b/vllm/transformers_utils/processors/internvl.py
@@ -12,7 +12,12 @@ import numpy.typing as npt
import torch
import torchvision.transforms as T
from PIL import Image
-from transformers import BatchFeature, TensorType
+from transformers import (
+ BaseVideoProcessor,
+ BatchFeature,
+ ImageProcessingMixin,
+ TensorType,
+)
from transformers.processing_utils import ProcessorMixin
from vllm.multimodal.image import convert_image_mode
@@ -215,7 +220,7 @@ def video_to_pixel_values_internvl(
return pixel_values
-class InternVLImageProcessor:
+class InternVLImageProcessor(ImageProcessingMixin):
def __init__(
self,
image_size: int,
@@ -312,7 +317,7 @@ class InternVLImageProcessor:
return BatchFeature(image_inputs, tensor_type=return_tensors)
-class InternVLVideoProcessor:
+class InternVLVideoProcessor(BaseVideoProcessor):
def __init__(
self,
image_size: int,
diff --git a/vllm/transformers_utils/processors/isaac.py b/vllm/transformers_utils/processors/isaac.py
index da548a8f1e..2d791df016 100644
--- a/vllm/transformers_utils/processors/isaac.py
+++ b/vllm/transformers_utils/processors/isaac.py
@@ -7,7 +7,12 @@ import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image
-from transformers import BatchFeature, ProcessorMixin, TensorType
+from transformers import (
+ BatchFeature,
+ ImageProcessingMixin,
+ ProcessorMixin,
+ TensorType,
+)
from transformers.processing_utils import ProcessingKwargs
from typing_extensions import Unpack
@@ -322,7 +327,7 @@ class IsaacProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-
}
-class IsaacImageProcessor:
+class IsaacImageProcessor(ImageProcessingMixin):
model_input_names = ["pixel_values", "image_grid_thw"]
def __init__(
diff --git a/vllm/transformers_utils/processors/minicpmo.py b/vllm/transformers_utils/processors/minicpmo.py
index d5e5750ca5..899e0402ba 100644
--- a/vllm/transformers_utils/processors/minicpmo.py
+++ b/vllm/transformers_utils/processors/minicpmo.py
@@ -268,7 +268,7 @@ class MiniCPMOProcessor(ProcessorMixin):
def batch_decode(self, *args, **kwargs):
"""
This method forwards all its arguments to LlamaTokenizerFast's
- [`~PreTrainedTokenizer.batch_decode`]. Please refer to the
+ [`~PythonBackend.batch_decode`]. Please refer to the
docstring of this method for more information.
"""
output_ids = args[0]
@@ -289,7 +289,7 @@ class MiniCPMOProcessor(ProcessorMixin):
def decode(self, *args, **kwargs):
"""
This method forwards all its arguments to LlamaTokenizerFast's
- [`~PreTrainedTokenizer.decode`]. Please refer to the docstring
+ [`~PythonBackend.decode`]. Please refer to the docstring
of this method for more information.
"""
result = args[0]
diff --git a/vllm/transformers_utils/processors/minicpmv.py b/vllm/transformers_utils/processors/minicpmv.py
index 91c3a8e479..712cbbb47a 100644
--- a/vllm/transformers_utils/processors/minicpmv.py
+++ b/vllm/transformers_utils/processors/minicpmv.py
@@ -56,14 +56,9 @@ class MiniCPMVProcessor(ProcessorMixin):
image_processor_class = "AutoImageProcessor"
tokenizer_class = "AutoTokenizer"
- def __init__(self, image_processor=None, tokenizer=None):
+ def __init__(self, image_processor=None, tokenizer=None, version=None):
super().__init__(image_processor, tokenizer)
- # Newer (transformers v5.7+) MiniCPM-V image processors, e.g.
- # MiniCPMV4_6ImageProcessor, no longer carry a `version` attribute.
- # Fall back to None instead of hard-crashing: `version` is only used
- # to special-case the 2.5 tokenization path in `_convert`, and any
- # value other than 2.5 takes the default branch anyway.
- self.version = getattr(image_processor, "version", None)
+ self.version = version
def __call__(
self,
@@ -100,7 +95,7 @@ class MiniCPMVProcessor(ProcessorMixin):
def batch_decode(self, *args, **kwargs):
"""
This method forwards all its arguments to LlamaTokenizerFast's
- [`~PreTrainedTokenizer.batch_decode`]. Please refer to the
+ [`~PythonBackend.batch_decode`]. Please refer to the
docstring of this method for more information.
"""
output_ids = args[0]
@@ -133,7 +128,7 @@ class MiniCPMVProcessor(ProcessorMixin):
def decode(self, *args, **kwargs):
"""
This method forwards all its arguments to LlamaTokenizerFast's
- [`~PreTrainedTokenizer.decode`]. Please refer to the docstring
+ [`~PythonBackend.decode`]. Please refer to the docstring
of this method for more information.
"""
result = args[0]
@@ -161,7 +156,7 @@ class MiniCPMVProcessor(ProcessorMixin):
def _convert(self, input_str, max_inp_length: int | None = None):
add_bos = getattr(self.tokenizer, "add_bos_token", False)
- if self.version == 2.5 or add_bos:
+ if self.version == (2, 5) or add_bos:
input_ids = self.tokenizer.encode(input_str)
else:
bos_id = getattr(
diff --git a/vllm/transformers_utils/processors/moondream3.py b/vllm/transformers_utils/processors/moondream3.py
index 289c40dd17..ae6833b200 100644
--- a/vllm/transformers_utils/processors/moondream3.py
+++ b/vllm/transformers_utils/processors/moondream3.py
@@ -195,7 +195,7 @@ class Moondream3Processor(ProcessorMixin):
The moondream3 model uses a custom tokenizer from 'moondream/starmie-v1'
instead of having tokenizer files in the model repo.
"""
- from transformers import AutoTokenizer, PreTrainedTokenizerFast
+ from transformers import AutoTokenizer, TokenizersBackend
from transformers.utils import cached_file
tokenizer = kwargs.pop("tokenizer", None)
@@ -237,7 +237,7 @@ class Moondream3Processor(ProcessorMixin):
"tokenizer.json",
**cached_file_kwargs,
)
- return PreTrainedTokenizerFast(
+ return TokenizersBackend(
tokenizer_file=tokenizer_file,
clean_up_tokenization_spaces=False,
)
diff --git a/vllm/transformers_utils/processors/openvla.py b/vllm/transformers_utils/processors/openvla.py
index 162f402383..e520f3b139 100644
--- a/vllm/transformers_utils/processors/openvla.py
+++ b/vllm/transformers_utils/processors/openvla.py
@@ -7,7 +7,7 @@ from typing import Any
import numpy as np
import torch
from PIL import Image
-from transformers.processing_utils import ProcessorMixin
+from transformers import ImageProcessingMixin, ProcessorMixin
IMAGENET_MEAN = np.array([0.484375, 0.455078125, 0.40625], dtype=np.float32)
IMAGENET_STD = np.array([0.228515625, 0.2236328125, 0.224609375], dtype=np.float32)
@@ -65,7 +65,7 @@ def preprocess_openvla_image(image: Any, image_size: int) -> torch.Tensor:
return torch.from_numpy(pixel_values)
-class OpenVLAImageProcessor:
+class OpenVLAImageProcessor(ImageProcessingMixin):
def __init__(self, *, image_size: int) -> None:
self.image_size = image_size
diff --git a/vllm/transformers_utils/processors/ovis.py b/vllm/transformers_utils/processors/ovis.py
index da80f24e75..907c68035f 100644
--- a/vllm/transformers_utils/processors/ovis.py
+++ b/vllm/transformers_utils/processors/ovis.py
@@ -417,14 +417,14 @@ class OvisProcessor(ProcessorMixin):
def batch_decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+ This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.batch_decode`]. Please
refer to the docstring of this method for more information.
"""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+ This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.decode`]. Please refer to
the docstring of this method for more information.
"""
return self.tokenizer.decode(*args, **kwargs)
diff --git a/vllm/transformers_utils/processors/pixtral.py b/vllm/transformers_utils/processors/pixtral.py
index c03360a2a5..588ad6b7fa 100644
--- a/vllm/transformers_utils/processors/pixtral.py
+++ b/vllm/transformers_utils/processors/pixtral.py
@@ -4,13 +4,13 @@ import torch
from mistral_common.protocol.instruct.chunk import ImageChunk
from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
from PIL import Image
-from transformers import BatchFeature, ProcessorMixin, TensorType
+from transformers import BatchFeature, ImageProcessingMixin, ProcessorMixin, TensorType
from transformers.image_utils import ImageInput
from vllm.tokenizers.mistral import MistralTokenizer
-class MistralCommonImageProcessor:
+class MistralCommonImageProcessor(ImageProcessingMixin):
"""
Provide a HF-compatible interface for
`mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
diff --git a/vllm/transformers_utils/processors/step3_vl.py b/vllm/transformers_utils/processors/step3_vl.py
index 71540f433f..8957a7c353 100644
--- a/vllm/transformers_utils/processors/step3_vl.py
+++ b/vllm/transformers_utils/processors/step3_vl.py
@@ -8,7 +8,12 @@ import torch
from PIL import Image
from torchvision import transforms
from torchvision.transforms.functional import InterpolationMode
-from transformers import BatchFeature, ProcessorMixin, TensorType
+from transformers import (
+ BatchFeature,
+ ImageProcessingMixin,
+ ProcessorMixin,
+ TensorType,
+)
from vllm.tokenizers import TokenizerLike
@@ -240,7 +245,7 @@ class ImagePatcher:
)
-class Step3VLImageProcessor:
+class Step3VLImageProcessor(ImageProcessingMixin):
def __init__(
self,
image_size: int = 728,
diff --git a/vllm/transformers_utils/processors/voxtral.py b/vllm/transformers_utils/processors/voxtral.py
index f67bfe9d2e..5403a7e47f 100644
--- a/vllm/transformers_utils/processors/voxtral.py
+++ b/vllm/transformers_utils/processors/voxtral.py
@@ -6,13 +6,18 @@ from math import ceil
import numpy as np
import torch
from mistral_common.tokens.tokenizers.audio import AudioEncoder
-from transformers import BatchFeature, ProcessorMixin, TensorType
+from transformers import (
+ BatchFeature,
+ ProcessorMixin,
+ SequenceFeatureExtractor,
+ TensorType,
+)
from transformers.audio_utils import AudioInput
from vllm.tokenizers.mistral import MistralTokenizer
-class MistralCommonFeatureExtractor:
+class MistralCommonFeatureExtractor(SequenceFeatureExtractor):
"""
Provide a HF-compatible interface for
`mistral_common.tokens.tokenizers.multimodal.AudioEncoder`.
diff --git a/vllm/v1/engine/detokenizer.py b/vllm/v1/engine/detokenizer.py
index 4700eecb59..50f14b9f96 100644
--- a/vllm/v1/engine/detokenizer.py
+++ b/vllm/v1/engine/detokenizer.py
@@ -6,7 +6,7 @@ import tokenizers
import tokenizers.decoders
from packaging import version
from tokenizers import Tokenizer
-from transformers import PreTrainedTokenizerFast
+from transformers import TokenizersBackend
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
@@ -57,7 +57,7 @@ class IncrementalDetokenizer:
# No tokenizer => skipping detokenization.
return IncrementalDetokenizer()
- if USE_FAST_DETOKENIZER and isinstance(tokenizer, PreTrainedTokenizerFast):
+ if USE_FAST_DETOKENIZER and isinstance(tokenizer, TokenizersBackend):
# Fast tokenizer => use tokenizers library DecodeStream.
return FastIncrementalDetokenizer(tokenizer, request)
@@ -165,7 +165,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
- def __init__(self, tokenizer: PreTrainedTokenizerFast, request: EngineCoreRequest):
+ def __init__(self, tokenizer: TokenizersBackend, request: EngineCoreRequest):
super().__init__(request)
sampling_params = request.sampling_params
diff --git a/vllm/v1/structured_output/backend_guidance.py b/vllm/v1/structured_output/backend_guidance.py
index 31178e9f24..19b2c76e9d 100644
--- a/vllm/v1/structured_output/backend_guidance.py
+++ b/vllm/v1/structured_output/backend_guidance.py
@@ -8,6 +8,7 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import torch
+from transformers import MistralCommonBackend
from vllm.logger import init_logger
from vllm.sampling_params import SamplingParams
@@ -95,6 +96,10 @@ class GuidanceBackend(StructuredOutputBackend):
if is_mistral_tokenizer(self.tokenizer):
self.ll_tokenizer = self.tokenizer.llg_tokenizer
+ elif isinstance(self.tokenizer, MistralCommonBackend):
+ from mistral_common.guidance.tokenizer import from_mistral_tokenizer
+
+ self.ll_tokenizer = from_mistral_tokenizer(self.tokenizer.tokenizer)
else:
self.ll_tokenizer = llguidance_hf.from_tokenizer(
self.tokenizer, max(self.vocab_size, len(self.tokenizer))