[Chore] Use Transformers v5 names and drop redundant processor use_fast (#58550)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-24 14:14:10 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 484c211fe8
commit d881812500
218 changed files with 869 additions and 813 deletions
@@ -7,7 +7,7 @@ from unittest.mock import MagicMock
import pytest
from tokenizers import Tokenizer, models, pre_tokenizers
from transformers import PreTrainedTokenizerFast
from transformers import TokenizersBackend
from vllm import PoolingParams
from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
@@ -70,7 +70,7 @@ def offline_processor_and_context():
backend = Tokenizer(models.WordLevel({"[UNK]": 0}, unk_token="[UNK]"))
backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
proc.tokenizer = PreTrainedTokenizerFast(tokenizer_object=backend)
proc.tokenizer = TokenizersBackend(tokenizer_object=backend)
proc.model_config = SimpleNamespace(max_model_len=1024, is_encoder_decoder=False)
proc.renderer = SimpleNamespace(
default_cmpl_tok_params=TokenizeParams(max_total_tokens=1024)