[Chore] Use Transformers v5 names and drop redundant processor use_fast (#58550)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
484c211fe8
commit
d881812500
@@ -7,7 +7,7 @@ from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from tokenizers import Tokenizer, models, pre_tokenizers
|
||||
from transformers import PreTrainedTokenizerFast
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
|
||||
@@ -70,7 +70,7 @@ def offline_processor_and_context():
|
||||
backend = Tokenizer(models.WordLevel({"[UNK]": 0}, unk_token="[UNK]"))
|
||||
backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
|
||||
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
|
||||
proc.tokenizer = PreTrainedTokenizerFast(tokenizer_object=backend)
|
||||
proc.tokenizer = TokenizersBackend(tokenizer_object=backend)
|
||||
proc.model_config = SimpleNamespace(max_model_len=1024, is_encoder_decoder=False)
|
||||
proc.renderer = SimpleNamespace(
|
||||
default_cmpl_tok_params=TokenizeParams(max_total_tokens=1024)
|
||||
|
||||
Reference in New Issue
Block a user