[CI/Build] Ensure compatability with Transformers v4.53 (#20541)

Signed-off-by: Isotr0py <2037008807@qq.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2025-07-11 20:53:07 -07:00
committed by GitHub
parent 11c0198615
commit 01cae37713
13 changed files with 74 additions and 38 deletions
+1 -7
View File
@@ -5,9 +5,7 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
import torch
from packaging.version import Version
from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
from transformers import __version__ as TRANSFORMERS_VERSION
from typing_extensions import TypeVar
from vllm.jsontree import JSONTree, json_map_leaves
@@ -137,13 +135,9 @@ class InputProcessingContext(InputContext):
/,
**kwargs: object,
) -> _P:
# Transformers 4.53.0 has issue with passing tokenizer to
# initialize processor. We disable it for this version.
# See: https://github.com/vllm-project/vllm/issues/20224
if Version(TRANSFORMERS_VERSION) != Version("4.53.0"):
kwargs["tokenizer"] = self.tokenizer
return super().get_hf_processor(
typ,
tokenizer=self.tokenizer,
**kwargs,
)