Bump Transformers version to 5.10.4 (#41359)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -12,7 +12,7 @@ from dataclasses import dataclass, field
|
||||
import aiohttp
|
||||
import huggingface_hub.constants
|
||||
from tqdm.asyncio import tqdm
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
from transformers import AutoTokenizer, PythonBackend, TokenizersBackend
|
||||
|
||||
# NOTE(simon): do not import vLLM here so the benchmark script
|
||||
# can run without vLLM installed.
|
||||
@@ -609,7 +609,7 @@ def get_tokenizer(
|
||||
tokenizer_mode: str = "auto",
|
||||
trust_remote_code: bool = False,
|
||||
**kwargs,
|
||||
) -> PreTrainedTokenizer | PreTrainedTokenizerFast:
|
||||
) -> PythonBackend | TokenizersBackend:
|
||||
if pretrained_model_name_or_path is not None and not os.path.exists(
|
||||
pretrained_model_name_or_path
|
||||
):
|
||||
|
||||
@@ -439,7 +439,7 @@ Additionally, to enable structured output, you'll need to create a new `Reasoner
|
||||
end_token: str = "</think>"
|
||||
|
||||
@classmethod
|
||||
def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner:
|
||||
def from_tokenizer(cls, tokenizer: PythonBackend) -> Reasoner:
|
||||
return cls(
|
||||
start_token_id=tokenizer.encode("<think>", add_special_tokens=False)[0],
|
||||
end_token_id=tokenizer.encode("</think>", add_special_tokens=False)[0],
|
||||
|
||||
@@ -19,7 +19,7 @@ Run:
|
||||
"""
|
||||
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedTokenizer
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer, PythonBackend
|
||||
|
||||
from vllm import LLM
|
||||
|
||||
@@ -34,7 +34,7 @@ def init_tokenizer_and_llm(model_name: str):
|
||||
|
||||
def get_prompt_embeds(
|
||||
chat: list[dict[str, str]],
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
tokenizer: PythonBackend,
|
||||
embedding_layer: torch.nn.Module,
|
||||
):
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
@@ -45,7 +45,7 @@ def get_prompt_embeds(
|
||||
|
||||
|
||||
def single_prompt_inference(
|
||||
llm: LLM, tokenizer: PreTrainedTokenizer, embedding_layer: torch.nn.Module
|
||||
llm: LLM, tokenizer: PythonBackend, embedding_layer: torch.nn.Module
|
||||
):
|
||||
chat = [{"role": "user", "content": "Please tell me about the capital of France."}]
|
||||
prompt_embeds = get_prompt_embeds(chat, tokenizer, embedding_layer)
|
||||
@@ -64,7 +64,7 @@ def single_prompt_inference(
|
||||
|
||||
|
||||
def batch_prompt_inference(
|
||||
llm: LLM, tokenizer: PreTrainedTokenizer, embedding_layer: torch.nn.Module
|
||||
llm: LLM, tokenizer: PythonBackend, embedding_layer: torch.nn.Module
|
||||
):
|
||||
chats = [
|
||||
[{"role": "user", "content": "Please tell me about the capital of France."}],
|
||||
|
||||
@@ -672,7 +672,7 @@ pathvalidate==3.2.1
|
||||
# via pytablewriter
|
||||
patsy==1.0.1
|
||||
# via statsmodels
|
||||
peft==0.18.1
|
||||
peft==0.19.1
|
||||
# via -r requirements/test/cuda.in
|
||||
perceptron==0.1.4
|
||||
# via -r requirements/test/cuda.in
|
||||
@@ -1159,7 +1159,7 @@ tqdm==4.67.3
|
||||
# segmentation-models-pytorch
|
||||
# sentence-transformers
|
||||
# transformers
|
||||
transformers==5.5.3
|
||||
transformers==5.10.4
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# -r requirements/test/cuda.in
|
||||
|
||||
@@ -20,7 +20,7 @@ httpx
|
||||
librosa # required for audio tests
|
||||
vector_quantize_pytorch # required for minicpmo_26 test
|
||||
vocos # required for minicpmo_26 test
|
||||
peft>=0.18.1 # required for phi-4-mm test
|
||||
peft>=0.19.1 # required for phi-4-mm test
|
||||
pqdm
|
||||
ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests
|
||||
sentence-transformers>=5.2.0 # required for embedding tests
|
||||
@@ -39,7 +39,7 @@ open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_
|
||||
datamodel_code_generator # required for minicpm3 test
|
||||
lm-eval[api]>=0.4.12 # required for model evaluation test
|
||||
mteb[bm25s]>=2, <3 # required for mteb test
|
||||
transformers==5.5.3
|
||||
transformers==5.10.4
|
||||
tokenizers==0.22.2
|
||||
schemathesis>=4.0.0 # Required for openai schema test.
|
||||
# quantization
|
||||
|
||||
@@ -755,7 +755,7 @@ pathvalidate==3.2.1
|
||||
# via pytablewriter
|
||||
patsy==1.0.1
|
||||
# via statsmodels
|
||||
peft==0.18.1
|
||||
peft==0.19.1
|
||||
# via -r requirements/test/cuda.in
|
||||
perceptron==0.1.4
|
||||
# via -r requirements/test/cuda.in
|
||||
@@ -1261,7 +1261,7 @@ tqdm==4.67.3
|
||||
# segmentation-models-pytorch
|
||||
# sentence-transformers
|
||||
# transformers
|
||||
transformers==5.5.3
|
||||
transformers==5.10.4
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
|
||||
@@ -29,7 +29,7 @@ opencv-python-headless >= 4.13.0 # required for video test
|
||||
datamodel_code_generator # required for minicpm3 test
|
||||
lm-eval[api]>=0.4.12 # required for model evaluation test
|
||||
mteb[bm25s]>=2, <3 # required for mteb test
|
||||
transformers==5.5.3
|
||||
transformers==5.10.4
|
||||
tokenizers==0.22.2
|
||||
schemathesis>=4.0.0 # Required for openai schema test.
|
||||
# quantization
|
||||
|
||||
@@ -19,7 +19,7 @@ httpx
|
||||
librosa # required for audio tests
|
||||
vector_quantize_pytorch # required for minicpmo_26 test
|
||||
vocos # required for minicpmo_26 test
|
||||
peft>=0.15.0 # required for phi-4-mm test
|
||||
peft>=0.19.1 # required for phi-4-mm test
|
||||
pqdm
|
||||
ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests
|
||||
sentence-transformers>=5.2.0 # required for embedding tests
|
||||
@@ -35,7 +35,7 @@ open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_
|
||||
datamodel_code_generator # required for minicpm3 test
|
||||
lm-eval[api]>=0.4.12 # required for model evaluation test
|
||||
mteb[bm25s]>=2, <3 # required for mteb test
|
||||
transformers==5.5.3
|
||||
transformers==5.10.4
|
||||
tokenizers==0.22.2
|
||||
schemathesis>=4.0.0 # Required for openai schema test
|
||||
# quantization
|
||||
|
||||
@@ -731,7 +731,7 @@ pathvalidate==3.3.1
|
||||
# via pytablewriter
|
||||
patsy==1.0.2
|
||||
# via statsmodels
|
||||
peft==0.18.1
|
||||
peft==0.19.1
|
||||
# via -r requirements/test/rocm.in
|
||||
perceptron==0.1.4
|
||||
# via -r requirements/test/rocm.in
|
||||
@@ -1218,7 +1218,7 @@ tqdm==4.67.3
|
||||
# sentence-transformers
|
||||
# tilelang
|
||||
# transformers
|
||||
transformers==5.5.3
|
||||
transformers==5.10.4
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
|
||||
@@ -17,6 +17,7 @@ accelerate
|
||||
arctic-inference
|
||||
lm_eval[api]>=0.4.12
|
||||
modelscope<1.38
|
||||
transformers==5.10.4
|
||||
|
||||
# --- Audio Processing ---
|
||||
librosa
|
||||
|
||||
@@ -940,10 +940,11 @@ tqdm==4.67.3
|
||||
# pqdm
|
||||
# sentence-transformers
|
||||
# transformers
|
||||
transformers==5.5.3
|
||||
transformers==5.10.4
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# -r requirements/test/xpu.in
|
||||
# compressed-tensors
|
||||
# sentence-transformers
|
||||
# xgrammar
|
||||
|
||||
@@ -378,7 +378,7 @@ mod tests {
|
||||
fs::write(dir.path().join("tokenizer.json"), "{}").expect("write tokenizer");
|
||||
fs::write(
|
||||
dir.path().join("tokenizer_config.json"),
|
||||
r#"{"tokenizer_class":"PreTrainedTokenizerFast"}"#,
|
||||
r#"{"tokenizer_class":"TokenizersBackend"}"#,
|
||||
)
|
||||
.expect("write tokenizer config");
|
||||
fs::write(dir.path().join("config.json"), "{}").expect("write config");
|
||||
|
||||
+2
-2
@@ -74,7 +74,7 @@ from torch._inductor.utils import fresh_cache
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
from transformers import PythonBackend, TokenizersBackend
|
||||
from transformers.generation.utils import GenerateOutput
|
||||
|
||||
|
||||
@@ -499,7 +499,7 @@ class HfRunner:
|
||||
self.model = model
|
||||
|
||||
if not skip_tokenizer_init:
|
||||
self.tokenizer: "PreTrainedTokenizer | PreTrainedTokenizerFast" = (
|
||||
self.tokenizer: "PythonBackend | TokenizersBackend" = (
|
||||
AutoTokenizer.from_pretrained(
|
||||
tokenizer_name or model_name,
|
||||
trust_remote_code=trust_remote_code,
|
||||
|
||||
@@ -369,7 +369,7 @@ async def test_chat_request(
|
||||
assert output.object == "list"
|
||||
assert len(output.data) == 1
|
||||
assert output.model == MODEL_NAME
|
||||
assert output.usage.prompt_tokens == 34
|
||||
assert output.usage.prompt_tokens == 33
|
||||
|
||||
# test continue_final_message
|
||||
response = requests.post(
|
||||
@@ -401,7 +401,7 @@ async def test_chat_request(
|
||||
assert output.object == "list"
|
||||
assert len(output.data) == 1
|
||||
assert output.model == MODEL_NAME
|
||||
assert output.usage.prompt_tokens == 36
|
||||
assert output.usage.prompt_tokens == 35
|
||||
|
||||
# test continue_final_message with add_generation_prompt
|
||||
response = requests.post(
|
||||
|
||||
@@ -906,7 +906,6 @@ VLM_TEST_SETTINGS = {
|
||||
max_model_len=4096,
|
||||
use_tokenizer_eos=True,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
hf_model_kwargs=model_utils.qianfan_ocr_hf_model_kwargs("baidu/Qianfan-OCR"),
|
||||
),
|
||||
"qwen2_vl": VLMTestInfo(
|
||||
models=["Qwen/Qwen2-VL-2B-Instruct"],
|
||||
|
||||
@@ -1492,94 +1492,3 @@ def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
|
||||
hf_model.model.generate = types.MethodType(_generate, hf_model.model)
|
||||
return hf_model
|
||||
|
||||
|
||||
def qianfan_ocr_hf_model_kwargs(model_name: str) -> dict:
|
||||
"""Return hf_model_kwargs with a patched config for QianfanOCR."""
|
||||
from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig
|
||||
|
||||
config = QianfanOCRConfig.from_pretrained(model_name)
|
||||
vc = config.vision_config
|
||||
if isinstance(vc.image_size, int):
|
||||
vc.image_size = (vc.image_size, vc.image_size)
|
||||
if isinstance(vc.patch_size, int):
|
||||
vc.patch_size = (vc.patch_size, vc.patch_size)
|
||||
return {"config": config}
|
||||
|
||||
|
||||
def qianfan_ocr_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
"""Patches an HfRunner instance to run QianfanOCR model inference.
|
||||
|
||||
QianfanOCR shares the same architecture as InternVLChatModel, so the
|
||||
patching logic mirrors ``internvl_patch_hf_runner``. The only difference
|
||||
is that we load the config via vllm's registered ``QianfanOCRConfig``
|
||||
instead of relying on ``trust_remote_code``.
|
||||
"""
|
||||
|
||||
class QianfanOCRProcessor:
|
||||
def __init__(self, hf_runner: HfRunner):
|
||||
self.tokenizer = hf_runner.tokenizer
|
||||
|
||||
from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig
|
||||
|
||||
self.config = QianfanOCRConfig.from_pretrained(hf_runner.model_name)
|
||||
self.vision_config = self.config.vision_config
|
||||
self.use_thumbnail = self.config.use_thumbnail
|
||||
self.min_num = self.config.min_dynamic_patch
|
||||
self.max_num = self.config.max_dynamic_patch
|
||||
self.image_size = self.vision_config.image_size
|
||||
|
||||
# Compute num_image_token from config instead of model attribute,
|
||||
# since the transformers-native model doesn't expose it.
|
||||
image_size = self.config.force_image_size or self.vision_config.image_size
|
||||
patch_size = self.vision_config.patch_size
|
||||
downsample_ratio = self.config.downsample_ratio
|
||||
self.num_image_token = int(
|
||||
(image_size // patch_size) ** 2 * (downsample_ratio**2)
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: str,
|
||||
images: PIL.Image.Image | list[PIL.Image.Image] = None,
|
||||
**kwargs,
|
||||
):
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
image_to_pixel_values_internvl,
|
||||
)
|
||||
|
||||
IMG_START = "<img>"
|
||||
IMG_END = "</img>"
|
||||
IMG_CONTEXT = "<IMG_CONTEXT>"
|
||||
|
||||
images = [images] if isinstance(images, PIL.Image.Image) else images
|
||||
pixel_values_list = [
|
||||
image_to_pixel_values_internvl(
|
||||
image,
|
||||
input_size=self.image_size,
|
||||
min_num=self.min_num,
|
||||
max_num=self.max_num,
|
||||
use_thumbnail=self.use_thumbnail,
|
||||
)
|
||||
for image in images
|
||||
]
|
||||
num_patches_list = [pv.shape[0] for pv in pixel_values_list]
|
||||
pixel_values = torch.cat(pixel_values_list, dim=0)
|
||||
|
||||
for num_patches in num_patches_list:
|
||||
context_tokens = IMG_CONTEXT * self.num_image_token * num_patches
|
||||
image_tokens = IMG_START + context_tokens + IMG_END
|
||||
text = text.replace("<image>", image_tokens, 1)
|
||||
|
||||
prompt = self.tokenizer(text, return_tensors="pt")
|
||||
prompt.update({"pixel_values": pixel_values})
|
||||
return prompt
|
||||
|
||||
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
|
||||
hf_model.model.img_context_token_id = img_context_token_id
|
||||
hf_model.processor = QianfanOCRProcessor(hf_model)
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.language_model.get_output_embeddings()
|
||||
)
|
||||
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
|
||||
return hf_model
|
||||
|
||||
@@ -202,9 +202,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"AfmoeForCausalLM": _HfExamplesInfo("arcee-ai/Trinity-Nano-Preview"),
|
||||
"ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"),
|
||||
"ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"),
|
||||
"ArcticForCausalLM": _HfExamplesInfo(
|
||||
"Snowflake/snowflake-arctic-instruct", trust_remote_code=True
|
||||
),
|
||||
"ArcticForCausalLM": _HfExamplesInfo("Snowflake/snowflake-arctic-instruct"),
|
||||
"AXK1ForCausalLM": _HfExamplesInfo("skt/A.X-K1", trust_remote_code=True),
|
||||
"BailingMoeForCausalLM": _HfExamplesInfo(
|
||||
"inclusionAI/Ling-lite-1.5", trust_remote_code=True
|
||||
|
||||
@@ -486,6 +486,7 @@ def dummy_hf_overrides(
|
||||
"Gemma3nForConditionalGeneration",
|
||||
"Gemma4ForCausalLM",
|
||||
"Gemma4ForConditionalGeneration",
|
||||
"Gemma4MTPModel",
|
||||
"DiffusionGemmaForBlockDiffusion",
|
||||
)
|
||||
else 1
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import _get_protocol_attrs # type: ignore
|
||||
import pytest
|
||||
from transformers import (
|
||||
PreTrainedTokenizerBase,
|
||||
PreTrainedTokenizerFast,
|
||||
TokenizersBackend,
|
||||
)
|
||||
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
@@ -24,7 +24,7 @@ def _assert_tokenizer_like(tokenizer: object):
|
||||
|
||||
def test_tokenizer_like_protocol():
|
||||
tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True)
|
||||
assert isinstance(tokenizer, PreTrainedTokenizerFast)
|
||||
assert isinstance(tokenizer, TokenizersBackend)
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
tokenizer = get_tokenizer(
|
||||
@@ -38,7 +38,7 @@ def test_tokenizer_like_protocol():
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
|
||||
# Verify it's a fast tokenizer (required for FastIncrementalDetokenizer)
|
||||
assert isinstance(tokenizer, PreTrainedTokenizerFast)
|
||||
assert isinstance(tokenizer, TokenizersBackend)
|
||||
assert "DSV32" in tokenizer.__class__.__name__
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from collections.abc import Generator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
from transformers import AutoTokenizer, PythonBackend, TokenizersBackend
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
@@ -153,13 +153,13 @@ def test_decode_streaming(
|
||||
spaces_between_special_tokens,
|
||||
fast,
|
||||
):
|
||||
if fast and not isinstance(tokenizer, PreTrainedTokenizerFast):
|
||||
if fast and not isinstance(tokenizer, TokenizersBackend):
|
||||
pytest.skip()
|
||||
|
||||
if skip_special_tokens and not spaces_between_special_tokens:
|
||||
pytest.skip()
|
||||
|
||||
if not fast and isinstance(tokenizer, PreTrainedTokenizerFast):
|
||||
if not fast and isinstance(tokenizer, TokenizersBackend):
|
||||
# Fix up inconsistency in fast/slow tokenizer behaviour.
|
||||
tokenizer.add_special_tokens(
|
||||
{
|
||||
@@ -173,7 +173,7 @@ def test_decode_streaming(
|
||||
|
||||
extra_decode_args = (
|
||||
{}
|
||||
if not isinstance(tokenizer, PreTrainedTokenizer)
|
||||
if not isinstance(tokenizer, PythonBackend)
|
||||
else {"spaces_between_special_tokens": spaces_between_special_tokens}
|
||||
)
|
||||
|
||||
@@ -225,7 +225,7 @@ def test_decode_streaming(
|
||||
@pytest.mark.parametrize("tokenizer_name", TOKENIZERS)
|
||||
@pytest.mark.parametrize("fast", (True, False))
|
||||
def test_oov_decode(tokenizer, fast):
|
||||
if fast and not isinstance(tokenizer, PreTrainedTokenizerFast):
|
||||
if fast and not isinstance(tokenizer, TokenizersBackend):
|
||||
pytest.skip()
|
||||
|
||||
decoded_text, out_ids = _run_incremental_decode(
|
||||
|
||||
@@ -7,14 +7,14 @@ from typing import TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
from transformers import PythonBackend, TokenizersBackend
|
||||
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.v1.engine import EngineCoreOutput, FinishReason
|
||||
from vllm.v1.metrics.stats import PrefillStats
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
|
||||
|
||||
GeneralTokenizerType: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast
|
||||
GeneralTokenizerType: TypeAlias = PythonBackend | TokenizersBackend
|
||||
|
||||
# Number of sample logprobs to request when testing sample logprobs
|
||||
NUM_SAMPLE_LOGPROBS_UNDER_TEST = 5
|
||||
@@ -193,7 +193,7 @@ def _create_random_top_token_test_matrix(
|
||||
|
||||
def decode_token(
|
||||
tok_id: int,
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
tokenizer: PythonBackend,
|
||||
) -> str:
|
||||
"""Reproduce the process of detokenizing a token for testing purposes.
|
||||
|
||||
@@ -210,7 +210,7 @@ def decode_token(
|
||||
def generate_dummy_sample_logprobs(
|
||||
sampled_tokens_list: list,
|
||||
num_logprobs: int,
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
tokenizer: PythonBackend,
|
||||
) -> list[tuple[list[int], list[float], int]]:
|
||||
"""Generate dummy sample logprobs
|
||||
|
||||
@@ -259,7 +259,7 @@ def generate_dummy_sample_logprobs(
|
||||
def generate_dummy_prompt_logprobs_tensors(
|
||||
prompt_tokens_list: list,
|
||||
num_logprobs: int,
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
tokenizer: PythonBackend,
|
||||
) -> LogprobsTensors:
|
||||
"""Generate dummy prompt logprobs tensors
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ from typing import Any
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PreTrainedTokenizerFast, WhisperFeatureExtractor
|
||||
from transformers import TokenizersBackend, WhisperFeatureExtractor
|
||||
from transformers.activations import get_activation
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.modeling_outputs import BaseModelOutput
|
||||
@@ -556,15 +556,15 @@ class FunAudioChatProcessingInfo(BaseProcessingInfo):
|
||||
return WhisperFeatureExtractor.from_pretrained(self.model_id)
|
||||
|
||||
@cached_property
|
||||
def speech_tokenizer(self) -> PreTrainedTokenizerFast:
|
||||
return PreTrainedTokenizerFast.from_pretrained(
|
||||
def speech_tokenizer(self) -> TokenizersBackend:
|
||||
return TokenizersBackend.from_pretrained(
|
||||
self.model_id, subfolder="speech_tokenizer"
|
||||
)
|
||||
|
||||
def get_feature_extractor(self) -> WhisperFeatureExtractor:
|
||||
return self.feature_extractor
|
||||
|
||||
def get_speech_tokenizer(self) -> PreTrainedTokenizerFast:
|
||||
def get_speech_tokenizer(self) -> TokenizersBackend:
|
||||
return self.speech_tokenizer
|
||||
|
||||
def get_data_parser(self):
|
||||
|
||||
@@ -552,6 +552,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
||||
vendored_processor = MiniCPMVProcessor(
|
||||
image_processor=hf_processor.image_processor,
|
||||
tokenizer=hf_processor.tokenizer,
|
||||
version=self.get_model_version(),
|
||||
)
|
||||
hf_processor = vendored_processor
|
||||
|
||||
|
||||
@@ -35,6 +35,8 @@ from vllm.multimodal.parse import ImageProcessorItems, ImageSize, VideoProcessor
|
||||
from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdateDetails,
|
||||
ResolvedPromptUpdate,
|
||||
_seq2text,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -408,6 +410,35 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
for modality, pattern in placeholders
|
||||
]
|
||||
|
||||
def _recompute_cached_prompt_update(
|
||||
self, cached_update: ResolvedPromptUpdate, new_item_idx: int
|
||||
) -> ResolvedPromptUpdate:
|
||||
new_update = super()._recompute_cached_prompt_update(
|
||||
cached_update, new_item_idx
|
||||
)
|
||||
# MiniCPM-V 4.6 prefixes video placeholders with `<image_id>{idx}</image_id>`
|
||||
# (the base class only rewrites the image modality).
|
||||
if cached_update.modality == "video":
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
|
||||
text = _seq2text(tokenizer, cached_update.content.full)
|
||||
prev_item_idx = cached_update.item_idx
|
||||
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_text(
|
||||
text.replace(
|
||||
f"{id_start}{prev_item_idx}{id_end}",
|
||||
f"{id_start}{new_item_idx}{id_end}",
|
||||
1,
|
||||
),
|
||||
video_token,
|
||||
)
|
||||
)
|
||||
return new_update
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs,
|
||||
|
||||
@@ -33,7 +33,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers import BatchFeature
|
||||
from transformers import BatchFeature, ProcessorMixin
|
||||
from transformers.models.qwen2_vl import Qwen2VLImageProcessor
|
||||
from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
|
||||
smart_resize as image_smart_resize,
|
||||
@@ -1191,34 +1191,31 @@ def _replace_video_token_placeholders(
|
||||
target: list[int],
|
||||
replacements: list[list[int]],
|
||||
) -> list[int]:
|
||||
"""Replace each 3-token video placeholder with its expanded sequence.
|
||||
"""Replace each video placeholder with its expanded sequence.
|
||||
|
||||
Args:
|
||||
prompt_ids: Token IDs of the original (unexpanded) prompt.
|
||||
target: 3-element list ``[vision_start_id, video_pad_id,
|
||||
vision_end_id]`` to search for.
|
||||
target: Token-ID sequence to search for. New-style
|
||||
(transformers>=5.10) processors expand only the bare
|
||||
`[video_pad_id]`; older ones expand the full
|
||||
`[vision_start_id, video_pad_id, vision_end_id]` triplet.
|
||||
replacements: Per-video expanded token sequences, in prompt order.
|
||||
|
||||
Returns:
|
||||
Token IDs with every placeholder triplet replaced.
|
||||
Token IDs with every placeholder replaced.
|
||||
"""
|
||||
result: list[int] = []
|
||||
repl_idx = 0
|
||||
i = 0
|
||||
n = len(prompt_ids)
|
||||
t0, t1, t2 = target
|
||||
k = len(target)
|
||||
num_repl = len(replacements)
|
||||
|
||||
while i < n:
|
||||
if (
|
||||
i + 2 < n
|
||||
and prompt_ids[i] == t0
|
||||
and prompt_ids[i + 1] == t1
|
||||
and prompt_ids[i + 2] == t2
|
||||
):
|
||||
if prompt_ids[i : i + k] == target:
|
||||
result.extend(replacements[repl_idx])
|
||||
repl_idx += 1
|
||||
i += 3
|
||||
i += k
|
||||
else:
|
||||
result.append(prompt_ids[i])
|
||||
i += 1
|
||||
@@ -1230,6 +1227,15 @@ def _replace_video_token_placeholders(
|
||||
|
||||
|
||||
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
|
||||
@staticmethod
|
||||
def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
|
||||
"""Transformers>=5.10 processors override `replace_video_token`
|
||||
to expand only the bare video token, keeping the prompt's outer
|
||||
`<|vision_start|>`/`<|vision_end|>` markers."""
|
||||
mixin_impl = getattr(ProcessorMixin, "replace_video_token", None)
|
||||
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
|
||||
return proc_impl is not None and proc_impl is not mixin_impl
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
@@ -1365,14 +1371,17 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
# Replace each placeholder triplet with pre-computed video tokens.
|
||||
# Replace each placeholder with pre-computed video tokens.
|
||||
if video_input_ids_lst:
|
||||
hf_config = self.info.get_hf_config()
|
||||
video_target = [
|
||||
hf_config.vision_start_token_id,
|
||||
hf_config.video_token_id,
|
||||
hf_config.vision_end_token_id,
|
||||
]
|
||||
if self._expands_only_video_token(self.info.get_hf_processor()):
|
||||
video_target = [hf_config.video_token_id]
|
||||
else:
|
||||
video_target = [
|
||||
hf_config.vision_start_token_id,
|
||||
hf_config.video_token_id,
|
||||
hf_config.vision_end_token_id,
|
||||
]
|
||||
input_ids = processed_outputs.pop("input_ids")
|
||||
if not isinstance(input_ids, list):
|
||||
input_ids = input_ids.tolist()
|
||||
@@ -1464,17 +1473,24 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
select_token_id=select_token_id,
|
||||
)
|
||||
|
||||
if self._expands_only_video_token(hf_processor):
|
||||
# transformers>=5.10 expands only the bare video_token
|
||||
video_target = hf_processor.video_token
|
||||
else:
|
||||
# Old-style processors expand the full placeholder
|
||||
# NOTE: We match string on purpose since searching sequence of
|
||||
# token ids takes more time.
|
||||
video_target = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=hf_processor.image_token,
|
||||
replacement=get_image_replacement_qwen3vl,
|
||||
),
|
||||
# NOTE: We match string on purpose since searching sequence of
|
||||
# token ids takes more time.
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target="<|vision_start|><|video_pad|><|vision_end|>",
|
||||
target=video_target,
|
||||
replacement=get_video_replacement_qwen3vl,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -276,13 +276,13 @@ class Base(
|
||||
- Any quantization config specific mappings
|
||||
"""
|
||||
self.hf_to_vllm_mapper = WeightsMapper()
|
||||
orig_to_new_renamings = self.hf_to_vllm_mapper.orig_to_new_renamings
|
||||
orig_to_new_renaming = self.hf_to_vllm_mapper.orig_to_new_renaming
|
||||
orig_to_new_regex = self.hf_to_vllm_mapper.orig_to_new_regex
|
||||
|
||||
for mapping in get_model_conversion_mapping(self.model):
|
||||
# Handle weights which have been renamed in Transformers
|
||||
if isinstance(mapping, WeightRenaming):
|
||||
orig_to_new_renamings.append(mapping)
|
||||
orig_to_new_renaming.append(mapping)
|
||||
# TODO: Handle WeightConverter to enable layer merging
|
||||
|
||||
# Handle unexpected weights which should be ignored
|
||||
|
||||
@@ -11,7 +11,6 @@ import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.nn.modules.module import register_module_module_registration_hook
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
@@ -33,6 +32,9 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import PretrainedConfig
|
||||
from transformers.conversion_mapping import WeightRenaming
|
||||
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -46,7 +48,7 @@ class WeightsMapper:
|
||||
|
||||
If a key maps to a value of `None`, the corresponding weight is ignored."""
|
||||
|
||||
orig_to_new_renamings: list[Any] = field(default_factory=list)
|
||||
orig_to_new_renaming: list["WeightRenaming"] = field(default_factory=list)
|
||||
orig_to_new_regex: Mapping[re.Pattern, str | None] = field(default_factory=dict)
|
||||
orig_to_new_substr: Mapping[str, str | None] = field(default_factory=dict)
|
||||
orig_to_new_stacked: Mapping[str, tuple[str, ShardId]] = field(default_factory=dict)
|
||||
@@ -56,9 +58,9 @@ class WeightsMapper:
|
||||
def __or__(self, other: "WeightsMapper") -> "WeightsMapper":
|
||||
"""Combine two `WeightsMapper`s by merging their mappings."""
|
||||
return WeightsMapper(
|
||||
orig_to_new_renamings=[
|
||||
*self.orig_to_new_renamings,
|
||||
*other.orig_to_new_renamings,
|
||||
orig_to_new_renaming=[
|
||||
*self.orig_to_new_renaming,
|
||||
*other.orig_to_new_renaming,
|
||||
],
|
||||
orig_to_new_regex={**self.orig_to_new_regex, **other.orig_to_new_regex},
|
||||
orig_to_new_substr={**self.orig_to_new_substr, **other.orig_to_new_substr},
|
||||
@@ -92,7 +94,7 @@ class WeightsMapper:
|
||||
"k_scale to v_scale"
|
||||
)
|
||||
|
||||
for renaming in self.orig_to_new_renamings:
|
||||
for renaming in self.orig_to_new_renaming:
|
||||
key, _ = renaming.rename_source_key(key)
|
||||
|
||||
for pattern, new_key in self.orig_to_new_regex.items():
|
||||
@@ -426,7 +428,7 @@ def init_vllm_registered_model(
|
||||
vllm_config: VllmConfig,
|
||||
*,
|
||||
prefix: str = "",
|
||||
hf_config: PretrainedConfig | None = None,
|
||||
hf_config: "PretrainedConfig | None" = None,
|
||||
architectures: list[str] | None = None,
|
||||
) -> nn.Module:
|
||||
"""
|
||||
|
||||
@@ -41,7 +41,7 @@ class ReasoningParser:
|
||||
|
||||
@cached_property
|
||||
def vocab(self) -> dict[str, int]:
|
||||
# NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab
|
||||
# NOTE: Only TokenizersBackend is guaranteed to have .vocab
|
||||
# whereas all tokenizers have .get_vocab()
|
||||
return self.model_tokenizer.get_vocab()
|
||||
|
||||
|
||||
+5
-13
@@ -228,20 +228,12 @@ def _try_get_processor_chat_template(
|
||||
if cache_key in _PROCESSOR_CHAT_TEMPLATES:
|
||||
return _PROCESSOR_CHAT_TEMPLATES[cache_key]
|
||||
|
||||
from transformers import (
|
||||
PreTrainedTokenizer,
|
||||
PreTrainedTokenizerFast,
|
||||
ProcessorMixin,
|
||||
)
|
||||
from transformers import ProcessorMixin, PythonBackend, TokenizersBackend
|
||||
|
||||
try:
|
||||
processor = cached_get_processor(
|
||||
tokenizer.name_or_path,
|
||||
processor_cls=(
|
||||
PreTrainedTokenizer,
|
||||
PreTrainedTokenizerFast,
|
||||
ProcessorMixin,
|
||||
),
|
||||
processor_cls=(PythonBackend, TokenizersBackend, ProcessorMixin),
|
||||
trust_remote_code=trust_remote_code,
|
||||
)
|
||||
if (
|
||||
@@ -619,15 +611,15 @@ _cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs)
|
||||
|
||||
@lru_cache
|
||||
def _get_hf_base_chat_template_params() -> frozenset[str]:
|
||||
from transformers import PreTrainedTokenizer
|
||||
from transformers import PythonBackend
|
||||
|
||||
# Get standard parameters from HuggingFace's base tokenizer class.
|
||||
# This dynamically extracts parameters from PreTrainedTokenizer's
|
||||
# This dynamically extracts parameters from PythonBackend's
|
||||
# apply_chat_template method, ensuring compatibility with tokenizers
|
||||
# that use **kwargs to receive standard parameters.
|
||||
|
||||
# Read signature from HF's base class - the single source of truth
|
||||
base_sig = inspect.signature(PreTrainedTokenizer.apply_chat_template)
|
||||
base_sig = inspect.signature(PythonBackend.apply_chat_template)
|
||||
|
||||
# Exclude VAR_KEYWORD (**kwargs) and VAR_POSITIONAL (*args) placeholders
|
||||
return frozenset(
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import copy
|
||||
from typing import Any
|
||||
|
||||
from transformers import PreTrainedTokenizerFast
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
|
||||
@@ -85,5 +85,5 @@ def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
class DeepseekV32Tokenizer(TokenizerLike):
|
||||
@classmethod
|
||||
def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:
|
||||
tokenizer = PreTrainedTokenizerFast.from_pretrained(*args, **kwargs)
|
||||
tokenizer = TokenizersBackend.from_pretrained(*args, **kwargs)
|
||||
return get_cached_tokenizer(get_deepseek_v32_tokenizer(tokenizer))
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import copy
|
||||
from typing import Any
|
||||
|
||||
from transformers import PreTrainedTokenizerFast
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
|
||||
@@ -92,5 +92,5 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
class DeepseekV4Tokenizer(TokenizerLike):
|
||||
@classmethod
|
||||
def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:
|
||||
tokenizer = PreTrainedTokenizerFast.from_pretrained(*args, **kwargs)
|
||||
tokenizer = TokenizersBackend.from_pretrained(*args, **kwargs)
|
||||
return get_cached_tokenizer(get_deepseek_v4_tokenizer(tokenizer))
|
||||
|
||||
+40
-5
@@ -6,13 +6,13 @@ import queue
|
||||
from pathlib import Path
|
||||
from typing import TypeAlias, TypeVar
|
||||
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
from transformers import AutoTokenizer, PythonBackend, TokenizersBackend
|
||||
|
||||
from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config
|
||||
|
||||
from .protocol import TokenizerLike
|
||||
|
||||
HfTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast
|
||||
HfTokenizer: TypeAlias = PythonBackend | TokenizersBackend
|
||||
_T = TypeVar("_T", bound=TokenizerLike)
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ class ThreadSafeHFTokenizerMixin:
|
||||
|
||||
def maybe_make_thread_pool(tokenizer: _T, copies: int = 1):
|
||||
"""
|
||||
If `tokenizer` is a `PreTrainedTokenizerFast`, modify the tokenizer
|
||||
If `tokenizer` is a `TokenizersBackend`, modify the tokenizer
|
||||
in-place to make the public interface thread-safe by routing calls
|
||||
through a deep-copied tokenizer pool.
|
||||
|
||||
@@ -34,14 +34,14 @@ def maybe_make_thread_pool(tokenizer: _T, copies: int = 1):
|
||||
methods like ``add_special_tokens`` or ``add_tokens``.
|
||||
- Adjacent method calls could happen on different deep copies.
|
||||
"""
|
||||
if not isinstance(tokenizer, PreTrainedTokenizerFast) or isinstance(
|
||||
if not isinstance(tokenizer, TokenizersBackend) or isinstance(
|
||||
tokenizer, ThreadSafeHFTokenizerMixin
|
||||
):
|
||||
return tokenizer
|
||||
|
||||
og_tokenizer = copy.copy(tokenizer)
|
||||
|
||||
tokenizer_pool: queue.Queue[PreTrainedTokenizerFast] = queue.Queue()
|
||||
tokenizer_pool: queue.Queue[TokenizersBackend] = queue.Queue()
|
||||
for _ in range(copies):
|
||||
tokenizer_pool.put(copy.deepcopy(og_tokenizer))
|
||||
|
||||
@@ -116,6 +116,16 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
tokenizer_all_special_tokens = tokenizer.all_special_tokens
|
||||
tokenizer_vocab = tokenizer.get_vocab()
|
||||
tokenizer_len = len(tokenizer)
|
||||
# The underlying tokenizer class could be MistralCommonBackend,
|
||||
# which does not implement is_fast in Transformers
|
||||
tokenizer_is_fast = getattr(tokenizer, "is_fast", True)
|
||||
|
||||
# MistralCommonBackend is tekken-backed and needs byte-fallback-aware tokenization.
|
||||
mistral_tekkenizer = None
|
||||
if getattr(getattr(tokenizer, "tokenizer", None), "instruct_tokenizer", None):
|
||||
from vllm.tokenizers.mistral import mistral_common_tekkenizer
|
||||
|
||||
mistral_tekkenizer = mistral_common_tekkenizer(tokenizer)
|
||||
|
||||
max_token_id = max(tokenizer_vocab.values())
|
||||
max_chars_per_token = max(len(tok) for tok in tokenizer_vocab)
|
||||
@@ -145,6 +155,31 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
def max_chars_per_token(self) -> int:
|
||||
return max_chars_per_token
|
||||
|
||||
@property
|
||||
def is_fast(self) -> bool:
|
||||
return tokenizer_is_fast
|
||||
|
||||
def convert_ids_to_tokens(self, ids, skip_special_tokens: bool = False):
|
||||
if mistral_tekkenizer is not None:
|
||||
from vllm.tokenizers.mistral import tekken_convert_ids_to_tokens
|
||||
|
||||
return tekken_convert_ids_to_tokens(mistral_tekkenizer, ids)
|
||||
return super().convert_ids_to_tokens(
|
||||
ids, skip_special_tokens=skip_special_tokens
|
||||
)
|
||||
|
||||
def convert_tokens_to_string(self, tokens: list[str]) -> str:
|
||||
if mistral_tekkenizer is not None:
|
||||
from vllm.tokenizers.mistral import tekken_convert_tokens_to_string
|
||||
|
||||
return tekken_convert_tokens_to_string(mistral_tekkenizer, tokens)
|
||||
try:
|
||||
return super().convert_tokens_to_string(tokens)
|
||||
except NotImplementedError:
|
||||
# The underlying tokenizer class could be MistralCommonBackend,
|
||||
# which does not implement convert_tokens_to_string in Transformers
|
||||
return "".join(tokens)
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return tokenizer_vocab
|
||||
|
||||
|
||||
@@ -188,6 +188,39 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int:
|
||||
return tokenizer.unk_id
|
||||
|
||||
|
||||
def mistral_common_tekkenizer(tokenizer: object) -> "Tekkenizer | None":
|
||||
"""Return the underlying `Tekkenizer` for a `MistralCommonBackend`."""
|
||||
mistral = getattr(tokenizer, "tokenizer", None)
|
||||
instruct = getattr(mistral, "instruct_tokenizer", None)
|
||||
tekken = getattr(instruct, "tokenizer", None)
|
||||
return tekken if isinstance(tekken, Tekkenizer) else None
|
||||
|
||||
|
||||
def tekken_convert_ids_to_tokens(
|
||||
tokenizer: "Tekkenizer", ids: Sequence[int]
|
||||
) -> list[str | bytes]:
|
||||
"""Convert ids to pieces, using raw `bytes` for byte-fallback tokens."""
|
||||
tokens: list[str | bytes] = [tokenizer.id_to_piece(i) for i in ids]
|
||||
if any("�" in t for t in tokens):
|
||||
tokens = [
|
||||
tokenizer.id_to_byte_piece(i, SpecialTokenPolicy.KEEP)
|
||||
if i >= tokenizer.num_special_tokens
|
||||
else tokenizer.decode([i], SpecialTokenPolicy.KEEP)
|
||||
for i in ids
|
||||
]
|
||||
return tokens
|
||||
|
||||
|
||||
def tekken_convert_tokens_to_string(
|
||||
tokenizer: "Tekkenizer", tokens: Sequence[str | bytes]
|
||||
) -> str:
|
||||
"""Reassemble pieces from `tekken_convert_ids_to_tokens` into text."""
|
||||
if any(isinstance(t, bytes) for t in tokens):
|
||||
ids = [_tekken_token_to_id(tokenizer, t) for t in tokens]
|
||||
return tokenizer.decode(ids, SpecialTokenPolicy.KEEP)
|
||||
return "".join(cast(Sequence[str], tokens))
|
||||
|
||||
|
||||
class MistralTokenizer(TokenizerLike):
|
||||
IS_MISTRAL_TOKENIZER = True # used by vllm.utils.mistral
|
||||
|
||||
@@ -453,14 +486,8 @@ class MistralTokenizer(TokenizerLike):
|
||||
if (t in to_decode_special_tokens or t not in self._special_tokens_set)
|
||||
]
|
||||
|
||||
if any(isinstance(t, bytes) for t in tokens):
|
||||
# we need to encode and decode all tokens again
|
||||
ids = [_tekken_token_to_id(self.tokenizer, t) for t in tokens]
|
||||
# We filtered unwanted special tokens before
|
||||
# so we can decode the rest.
|
||||
decoded = self.tokenizer.decode(ids, SpecialTokenPolicy.KEEP)
|
||||
else:
|
||||
decoded = "".join(tokens)
|
||||
# We filtered unwanted special tokens before so we can decode the rest.
|
||||
decoded = tekken_convert_tokens_to_string(self.tokenizer, tokens)
|
||||
else:
|
||||
# make sure certain special tokens like Tool calls are
|
||||
# not decoded
|
||||
|
||||
@@ -32,6 +32,7 @@ logger = init_logger(__name__)
|
||||
# - Add model type to MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS in transformers (better)
|
||||
# - Fix tokenizer_class on the hub for the affected models (best)
|
||||
_MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS: set[str] = {
|
||||
"internlm2",
|
||||
"step3_vl",
|
||||
"step3p7",
|
||||
"unlimited-ocr",
|
||||
|
||||
@@ -112,7 +112,7 @@ class ToolParser:
|
||||
|
||||
@cached_property
|
||||
def vocab(self) -> dict[str, int]:
|
||||
# NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab
|
||||
# NOTE: Only TokenizersBackend is guaranteed to have .vocab
|
||||
# whereas all tokenizers have .get_vocab()
|
||||
return self.model_tokenizer.get_vocab()
|
||||
|
||||
|
||||
@@ -71,6 +71,7 @@ class LazyConfigDict(dict):
|
||||
|
||||
_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
afmoe="AfmoeConfig",
|
||||
arctic="ArcticConfig",
|
||||
bagel="BagelConfig",
|
||||
umm="CheersConfig",
|
||||
chatglm="ChatGLMConfig",
|
||||
|
||||
@@ -16,6 +16,7 @@ import importlib
|
||||
|
||||
_CLASS_TO_MODULE: dict[str, str] = {
|
||||
"AfmoeConfig": "vllm.transformers_utils.configs.afmoe",
|
||||
"ArcticConfig": "vllm.transformers_utils.configs.arctic",
|
||||
"AXK1Config": "vllm.transformers_utils.configs.AXK1",
|
||||
"BagelConfig": "vllm.transformers_utils.configs.bagel",
|
||||
"CheersConfig": "vllm.transformers_utils.configs.cheers",
|
||||
@@ -92,6 +93,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
|
||||
__all__ = [
|
||||
"AfmoeConfig",
|
||||
"ArcticConfig",
|
||||
"AXK1Config",
|
||||
"BagelConfig",
|
||||
"CheersConfig",
|
||||
|
||||
@@ -232,7 +232,7 @@ class FireRedLIDProcessor(ProcessorMixin):
|
||||
"""
|
||||
|
||||
feature_extractor_class = "FireRedLIDFeatureExtractor"
|
||||
tokenizer_class = ("PreTrainedTokenizer", "PreTrainedTokenizerFast")
|
||||
tokenizer_class = ("PythonBackend", "TokenizersBackend")
|
||||
|
||||
def __init__(self, feature_extractor, tokenizer):
|
||||
super().__init__(feature_extractor, tokenizer)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
# Adapted from
|
||||
# https://github.com/zai-org/CogAgent
|
||||
from transformers import PreTrainedTokenizer
|
||||
from transformers import PythonBackend
|
||||
from transformers.image_processing_utils_fast import BaseImageProcessorFast
|
||||
from transformers.image_utils import PILImageResampling
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
@@ -30,7 +30,7 @@ class GLM4VProcessor(ProcessorMixin):
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: GLM4VImageProcessorFast,
|
||||
tokenizer: PreTrainedTokenizer,
|
||||
tokenizer: PythonBackend,
|
||||
) -> None:
|
||||
self.image_processor = image_processor
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
@@ -12,7 +12,12 @@ import numpy.typing as npt
|
||||
import torch
|
||||
import torchvision.transforms as T
|
||||
from PIL import Image
|
||||
from transformers import BatchFeature, TensorType
|
||||
from transformers import (
|
||||
BaseVideoProcessor,
|
||||
BatchFeature,
|
||||
ImageProcessingMixin,
|
||||
TensorType,
|
||||
)
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
@@ -215,7 +220,7 @@ def video_to_pixel_values_internvl(
|
||||
return pixel_values
|
||||
|
||||
|
||||
class InternVLImageProcessor:
|
||||
class InternVLImageProcessor(ImageProcessingMixin):
|
||||
def __init__(
|
||||
self,
|
||||
image_size: int,
|
||||
@@ -312,7 +317,7 @@ class InternVLImageProcessor:
|
||||
return BatchFeature(image_inputs, tensor_type=return_tensors)
|
||||
|
||||
|
||||
class InternVLVideoProcessor:
|
||||
class InternVLVideoProcessor(BaseVideoProcessor):
|
||||
def __init__(
|
||||
self,
|
||||
image_size: int,
|
||||
|
||||
@@ -7,7 +7,12 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from PIL import Image
|
||||
from transformers import BatchFeature, ProcessorMixin, TensorType
|
||||
from transformers import (
|
||||
BatchFeature,
|
||||
ImageProcessingMixin,
|
||||
ProcessorMixin,
|
||||
TensorType,
|
||||
)
|
||||
from transformers.processing_utils import ProcessingKwargs
|
||||
from typing_extensions import Unpack
|
||||
|
||||
@@ -322,7 +327,7 @@ class IsaacProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-
|
||||
}
|
||||
|
||||
|
||||
class IsaacImageProcessor:
|
||||
class IsaacImageProcessor(ImageProcessingMixin):
|
||||
model_input_names = ["pixel_values", "image_grid_thw"]
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -268,7 +268,7 @@ class MiniCPMOProcessor(ProcessorMixin):
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to LlamaTokenizerFast's
|
||||
[`~PreTrainedTokenizer.batch_decode`]. Please refer to the
|
||||
[`~PythonBackend.batch_decode`]. Please refer to the
|
||||
docstring of this method for more information.
|
||||
"""
|
||||
output_ids = args[0]
|
||||
@@ -289,7 +289,7 @@ class MiniCPMOProcessor(ProcessorMixin):
|
||||
def decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to LlamaTokenizerFast's
|
||||
[`~PreTrainedTokenizer.decode`]. Please refer to the docstring
|
||||
[`~PythonBackend.decode`]. Please refer to the docstring
|
||||
of this method for more information.
|
||||
"""
|
||||
result = args[0]
|
||||
|
||||
@@ -56,14 +56,9 @@ class MiniCPMVProcessor(ProcessorMixin):
|
||||
image_processor_class = "AutoImageProcessor"
|
||||
tokenizer_class = "AutoTokenizer"
|
||||
|
||||
def __init__(self, image_processor=None, tokenizer=None):
|
||||
def __init__(self, image_processor=None, tokenizer=None, version=None):
|
||||
super().__init__(image_processor, tokenizer)
|
||||
# Newer (transformers v5.7+) MiniCPM-V image processors, e.g.
|
||||
# MiniCPMV4_6ImageProcessor, no longer carry a `version` attribute.
|
||||
# Fall back to None instead of hard-crashing: `version` is only used
|
||||
# to special-case the 2.5 tokenization path in `_convert`, and any
|
||||
# value other than 2.5 takes the default branch anyway.
|
||||
self.version = getattr(image_processor, "version", None)
|
||||
self.version = version
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
@@ -100,7 +95,7 @@ class MiniCPMVProcessor(ProcessorMixin):
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to LlamaTokenizerFast's
|
||||
[`~PreTrainedTokenizer.batch_decode`]. Please refer to the
|
||||
[`~PythonBackend.batch_decode`]. Please refer to the
|
||||
docstring of this method for more information.
|
||||
"""
|
||||
output_ids = args[0]
|
||||
@@ -133,7 +128,7 @@ class MiniCPMVProcessor(ProcessorMixin):
|
||||
def decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to LlamaTokenizerFast's
|
||||
[`~PreTrainedTokenizer.decode`]. Please refer to the docstring
|
||||
[`~PythonBackend.decode`]. Please refer to the docstring
|
||||
of this method for more information.
|
||||
"""
|
||||
result = args[0]
|
||||
@@ -161,7 +156,7 @@ class MiniCPMVProcessor(ProcessorMixin):
|
||||
|
||||
def _convert(self, input_str, max_inp_length: int | None = None):
|
||||
add_bos = getattr(self.tokenizer, "add_bos_token", False)
|
||||
if self.version == 2.5 or add_bos:
|
||||
if self.version == (2, 5) or add_bos:
|
||||
input_ids = self.tokenizer.encode(input_str)
|
||||
else:
|
||||
bos_id = getattr(
|
||||
|
||||
@@ -195,7 +195,7 @@ class Moondream3Processor(ProcessorMixin):
|
||||
The moondream3 model uses a custom tokenizer from 'moondream/starmie-v1'
|
||||
instead of having tokenizer files in the model repo.
|
||||
"""
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerFast
|
||||
from transformers import AutoTokenizer, TokenizersBackend
|
||||
from transformers.utils import cached_file
|
||||
|
||||
tokenizer = kwargs.pop("tokenizer", None)
|
||||
@@ -237,7 +237,7 @@ class Moondream3Processor(ProcessorMixin):
|
||||
"tokenizer.json",
|
||||
**cached_file_kwargs,
|
||||
)
|
||||
return PreTrainedTokenizerFast(
|
||||
return TokenizersBackend(
|
||||
tokenizer_file=tokenizer_file,
|
||||
clean_up_tokenization_spaces=False,
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Any
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
from transformers import ImageProcessingMixin, ProcessorMixin
|
||||
|
||||
IMAGENET_MEAN = np.array([0.484375, 0.455078125, 0.40625], dtype=np.float32)
|
||||
IMAGENET_STD = np.array([0.228515625, 0.2236328125, 0.224609375], dtype=np.float32)
|
||||
@@ -65,7 +65,7 @@ def preprocess_openvla_image(image: Any, image_size: int) -> torch.Tensor:
|
||||
return torch.from_numpy(pixel_values)
|
||||
|
||||
|
||||
class OpenVLAImageProcessor:
|
||||
class OpenVLAImageProcessor(ImageProcessingMixin):
|
||||
def __init__(self, *, image_size: int) -> None:
|
||||
self.image_size = image_size
|
||||
|
||||
|
||||
@@ -417,14 +417,14 @@ class OvisProcessor(ProcessorMixin):
|
||||
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
|
||||
This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.batch_decode`]. Please
|
||||
refer to the docstring of this method for more information.
|
||||
"""
|
||||
return self.tokenizer.batch_decode(*args, **kwargs)
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
|
||||
This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.decode`]. Please refer to
|
||||
the docstring of this method for more information.
|
||||
"""
|
||||
return self.tokenizer.decode(*args, **kwargs)
|
||||
|
||||
@@ -4,13 +4,13 @@ import torch
|
||||
from mistral_common.protocol.instruct.chunk import ImageChunk
|
||||
from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
|
||||
from PIL import Image
|
||||
from transformers import BatchFeature, ProcessorMixin, TensorType
|
||||
from transformers import BatchFeature, ImageProcessingMixin, ProcessorMixin, TensorType
|
||||
from transformers.image_utils import ImageInput
|
||||
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
|
||||
class MistralCommonImageProcessor:
|
||||
class MistralCommonImageProcessor(ImageProcessingMixin):
|
||||
"""
|
||||
Provide a HF-compatible interface for
|
||||
`mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
|
||||
|
||||
@@ -8,7 +8,12 @@ import torch
|
||||
from PIL import Image
|
||||
from torchvision import transforms
|
||||
from torchvision.transforms.functional import InterpolationMode
|
||||
from transformers import BatchFeature, ProcessorMixin, TensorType
|
||||
from transformers import (
|
||||
BatchFeature,
|
||||
ImageProcessingMixin,
|
||||
ProcessorMixin,
|
||||
TensorType,
|
||||
)
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
@@ -240,7 +245,7 @@ class ImagePatcher:
|
||||
)
|
||||
|
||||
|
||||
class Step3VLImageProcessor:
|
||||
class Step3VLImageProcessor(ImageProcessingMixin):
|
||||
def __init__(
|
||||
self,
|
||||
image_size: int = 728,
|
||||
|
||||
@@ -6,13 +6,18 @@ from math import ceil
|
||||
import numpy as np
|
||||
import torch
|
||||
from mistral_common.tokens.tokenizers.audio import AudioEncoder
|
||||
from transformers import BatchFeature, ProcessorMixin, TensorType
|
||||
from transformers import (
|
||||
BatchFeature,
|
||||
ProcessorMixin,
|
||||
SequenceFeatureExtractor,
|
||||
TensorType,
|
||||
)
|
||||
from transformers.audio_utils import AudioInput
|
||||
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
|
||||
class MistralCommonFeatureExtractor:
|
||||
class MistralCommonFeatureExtractor(SequenceFeatureExtractor):
|
||||
"""
|
||||
Provide a HF-compatible interface for
|
||||
`mistral_common.tokens.tokenizers.multimodal.AudioEncoder`.
|
||||
|
||||
@@ -6,7 +6,7 @@ import tokenizers
|
||||
import tokenizers.decoders
|
||||
from packaging import version
|
||||
from tokenizers import Tokenizer
|
||||
from transformers import PreTrainedTokenizerFast
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
@@ -57,7 +57,7 @@ class IncrementalDetokenizer:
|
||||
# No tokenizer => skipping detokenization.
|
||||
return IncrementalDetokenizer()
|
||||
|
||||
if USE_FAST_DETOKENIZER and isinstance(tokenizer, PreTrainedTokenizerFast):
|
||||
if USE_FAST_DETOKENIZER and isinstance(tokenizer, TokenizersBackend):
|
||||
# Fast tokenizer => use tokenizers library DecodeStream.
|
||||
return FastIncrementalDetokenizer(tokenizer, request)
|
||||
|
||||
@@ -165,7 +165,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
|
||||
|
||||
|
||||
class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
def __init__(self, tokenizer: PreTrainedTokenizerFast, request: EngineCoreRequest):
|
||||
def __init__(self, tokenizer: TokenizersBackend, request: EngineCoreRequest):
|
||||
super().__init__(request)
|
||||
|
||||
sampling_params = request.sampling_params
|
||||
|
||||
@@ -8,6 +8,7 @@ from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from transformers import MistralCommonBackend
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
@@ -95,6 +96,10 @@ class GuidanceBackend(StructuredOutputBackend):
|
||||
|
||||
if is_mistral_tokenizer(self.tokenizer):
|
||||
self.ll_tokenizer = self.tokenizer.llg_tokenizer
|
||||
elif isinstance(self.tokenizer, MistralCommonBackend):
|
||||
from mistral_common.guidance.tokenizer import from_mistral_tokenizer
|
||||
|
||||
self.ll_tokenizer = from_mistral_tokenizer(self.tokenizer.tokenizer)
|
||||
else:
|
||||
self.ll_tokenizer = llguidance_hf.from_tokenizer(
|
||||
self.tokenizer, max(self.vocab_size, len(self.tokenizer))
|
||||
|
||||
Reference in New Issue
Block a user